| author |  Andrew Kelley <andrew@ziglang.org> 2023-01-10 20:21:58-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2023-01-11 15:39:49-08:00 |
| log | cfcf9771c1bde357ad64d81cda9d61ba72d80b15 |
| tree | 143a7434666e186bc79989c9108fdc3ab1c16741 |
| parent | a0f2e6a29f4d5c084a248d24b25fae9f30707001 |
zig build: support dependencies
The `zig build` command now makes `@import("@dependencies")` available
to the build runner package. It contains all the dependencies in a
generated file that looks something like this:
```zig
pub const imports = struct {
pub const foo = @import("foo");
pub const @"bar.baz" = @import("bar.baz");
};
pub const build_root = struct {
pub const foo = "<path>";
pub const @"bar.baz" = "<path>";
};
```
The build runner exports this import so that `std.build.Builder` can
access it. `std.build.Builder` uses it to implement the new `dependency`
function which can be used like so:
```zig
const libz_dep = b.dependency("libz", .{});
const libmp3lame_dep = b.dependency("libmp3lame", .{});
// ...
lib.linkLibrary(libz_dep.artifact("z"));
lib.linkLibrary(libmp3lame_dep.artifact("mp3lame"));
```
The `dependency` function calls the build.zig file of the dependency as
a child Builder, and then can be ransacked for its build steps via the
`artifact` function.
This commit also renames `dependency.id` to `dependency.name` in the
`build.zig.ini` file.4 files changed, 297 insertions(+), 57 deletions(-)
lib/build_runner.zig+4-10
| ... | @@ -9,6 +9,8 @@ const process = std.process; | ... | @@ -9,6 +9,8 @@ const process = std.process; |
| 9 | const ArrayList = std.ArrayList; | 9 | const ArrayList = std.ArrayList; |
| 10 | const File = std.fs.File; | 10 | const File = std.fs.File; |
| 11 | 11 | ||
| 12 | pub const dependencies = @import("@dependencies"); | ||
| 13 | |||
| 12 | pub fn main() !void { | 14 | pub fn main() !void { |
| 13 | // Here we use an ArenaAllocator backed by a DirectAllocator because a build is a short-lived, | 15 | // Here we use an ArenaAllocator backed by a DirectAllocator because a build is a short-lived, |
| 14 | // one shot program. We don't need to waste time freeing memory and finding places to squish | 16 | // one shot program. We don't need to waste time freeing memory and finding places to squish |
| ... | @@ -207,7 +209,7 @@ pub fn main() !void { | ... | @@ -207,7 +209,7 @@ pub fn main() !void { |
| 207 | 209 | ||
| 208 | builder.debug_log_scopes = debug_log_scopes.items; | 210 | builder.debug_log_scopes = debug_log_scopes.items; |
| 209 | builder.resolveInstallPrefix(install_prefix, dir_list); | 211 | builder.resolveInstallPrefix(install_prefix, dir_list); |
| 210 | try runBuild(builder); | 212 | try builder.runBuild(root); |
| 211 | 213 | ||
| 212 | if (builder.validateUserInputDidItFail()) | 214 | if (builder.validateUserInputDidItFail()) |
| 213 | return usageAndErr(builder, true, stderr_stream); | 215 | return usageAndErr(builder, true, stderr_stream); |
| ... | @@ -223,19 +225,11 @@ pub fn main() !void { | ... | @@ -223,19 +225,11 @@ pub fn main() !void { |
| 223 | }; | 225 | }; |
| 224 | } | 226 | } |
| 225 | 227 | ||
| 226 | fn runBuild(builder: *Builder) anyerror!void { | ||
| 227 | switch (@typeInfo(@typeInfo(@TypeOf(root.build)).Fn.return_type.?)) { | ||
| 228 | .Void => root.build(builder), | ||
| 229 | .ErrorUnion => try root.build(builder), | ||
| 230 | else => @compileError("expected return type of build to be 'void' or '!void'"), | ||
| 231 | } | ||
| 232 | } | ||
| 233 | |||
| 234 | fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void { | 228 | fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void { |
| 235 | // run the build script to collect the options | 229 | // run the build script to collect the options |
| 236 | if (!already_ran_build) { | 230 | if (!already_ran_build) { |
| 237 | builder.resolveInstallPrefix(null, .{}); | 231 | builder.resolveInstallPrefix(null, .{}); |
| 238 | try runBuild(builder); | 232 | try builder.runBuild(root); |
| 239 | } | 233 | } |
| 240 | 234 | ||
| 241 | try out_stream.print( | 235 | try out_stream.print( |
lib/std/build.zig+152-2
| ... | @@ -69,13 +69,15 @@ pub const Builder = struct { | ... | @@ -69,13 +69,15 @@ pub const Builder = struct { |
| 69 | search_prefixes: ArrayList([]const u8), | 69 | search_prefixes: ArrayList([]const u8), |
| 70 | libc_file: ?[]const u8 = null, | 70 | libc_file: ?[]const u8 = null, |
| 71 | installed_files: ArrayList(InstalledFile), | 71 | installed_files: ArrayList(InstalledFile), |
| 72 | /// Path to the directory containing build.zig. | ||
| 72 | build_root: []const u8, | 73 | build_root: []const u8, |
| 73 | cache_root: []const u8, | 74 | cache_root: []const u8, |
| 74 | global_cache_root: []const u8, | 75 | global_cache_root: []const u8, |
| 75 | release_mode: ?std.builtin.Mode, | 76 | release_mode: ?std.builtin.Mode, |
| 76 | is_release: bool, | 77 | is_release: bool, |
| 78 | /// zig lib dir | ||
| 77 | override_lib_dir: ?[]const u8, | 79 | override_lib_dir: ?[]const u8, |
| 78 | vcpkg_root: VcpkgRoot, | 80 | vcpkg_root: VcpkgRoot = .unattempted, |
| 79 | pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, | 81 | pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, |
| 80 | args: ?[][]const u8 = null, | 82 | args: ?[][]const u8 = null, |
| 81 | debug_log_scopes: []const []const u8 = &.{}, | 83 | debug_log_scopes: []const []const u8 = &.{}, |
| ... | @@ -100,6 +102,8 @@ pub const Builder = struct { | ... | @@ -100,6 +102,8 @@ pub const Builder = struct { |
| 100 | /// Information about the native target. Computed before build() is invoked. | 102 | /// Information about the native target. Computed before build() is invoked. |
| 101 | host: NativeTargetInfo, | 103 | host: NativeTargetInfo, |
| 102 | 104 | ||
| 105 | dep_prefix: []const u8 = "", | ||
| 106 | |||
| 103 | pub const ExecError = error{ | 107 | pub const ExecError = error{ |
| 104 | ReadFailure, | 108 | ReadFailure, |
| 105 | ExitCodeFailure, | 109 | ExitCodeFailure, |
| ... | @@ -223,7 +227,6 @@ pub const Builder = struct { | ... | @@ -223,7 +227,6 @@ pub const Builder = struct { |
| 223 | .is_release = false, | 227 | .is_release = false, |
| 224 | .override_lib_dir = null, | 228 | .override_lib_dir = null, |
| 225 | .install_path = undefined, | 229 | .install_path = undefined, |
| 226 | .vcpkg_root = VcpkgRoot{ .unattempted = {} }, | ||
| 227 | .args = null, | 230 | .args = null, |
| 228 | .host = host, | 231 | .host = host, |
| 229 | }; | 232 | }; |
| ... | @@ -233,6 +236,89 @@ pub const Builder = struct { | ... | @@ -233,6 +236,89 @@ pub const Builder = struct { |
| 233 | return self; | 236 | return self; |
| 234 | } | 237 | } |
| 235 | 238 | ||
| 239 | fn createChild( | ||
| 240 | parent: *Builder, | ||
| 241 | dep_name: []const u8, | ||
| 242 | build_root: []const u8, | ||
| 243 | args: anytype, | ||
| 244 | ) !*Builder { | ||
| 245 | const child = try createChildOnly(parent, dep_name, build_root); | ||
| 246 | try applyArgs(child, args); | ||
| 247 | return child; | ||
| 248 | } | ||
| 249 | |||
| 250 | fn createChildOnly(parent: *Builder, dep_name: []const u8, build_root: []const u8) !*Builder { | ||
| 251 | const allocator = parent.allocator; | ||
| 252 | const child = try allocator.create(Builder); | ||
| 253 | child.* = .{ | ||
| 254 | .allocator = allocator, | ||
| 255 | .install_tls = .{ | ||
| 256 | .step = Step.initNoOp(.top_level, "install", allocator), | ||
| 257 | .description = "Copy build artifacts to prefix path", | ||
| 258 | }, | ||
| 259 | .uninstall_tls = .{ | ||
| 260 | .step = Step.init(.top_level, "uninstall", allocator, makeUninstall), | ||
| 261 | .description = "Remove build artifacts from prefix path", | ||
| 262 | }, | ||
| 263 | .user_input_options = UserInputOptionsMap.init(allocator), | ||
| 264 | .available_options_map = AvailableOptionsMap.init(allocator), | ||
| 265 | .available_options_list = ArrayList(AvailableOption).init(allocator), | ||
| 266 | .verbose = parent.verbose, | ||
| 267 | .verbose_link = parent.verbose_link, | ||
| 268 | .verbose_cc = parent.verbose_cc, | ||
| 269 | .verbose_air = parent.verbose_air, | ||
| 270 | .verbose_llvm_ir = parent.verbose_llvm_ir, | ||
| 271 | .verbose_cimport = parent.verbose_cimport, | ||
| 272 | .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features, | ||
| 273 | .prominent_compile_errors = parent.prominent_compile_errors, | ||
| 274 | .color = parent.color, | ||
| 275 | .reference_trace = parent.reference_trace, | ||
| 276 | .invalid_user_input = false, | ||
| 277 | .zig_exe = parent.zig_exe, | ||
| 278 | .default_step = undefined, | ||
| 279 | .env_map = parent.env_map, | ||
| 280 | .top_level_steps = ArrayList(*TopLevelStep).init(allocator), | ||
| 281 | .install_prefix = undefined, | ||
| 282 | .dest_dir = parent.dest_dir, | ||
| 283 | .lib_dir = parent.lib_dir, | ||
| 284 | .exe_dir = parent.exe_dir, | ||
| 285 | .h_dir = parent.h_dir, | ||
| 286 | .install_path = parent.install_path, | ||
| 287 | .sysroot = parent.sysroot, | ||
| 288 | .search_prefixes = ArrayList([]const u8).init(allocator), | ||
| 289 | .libc_file = parent.libc_file, | ||
| 290 | .installed_files = ArrayList(InstalledFile).init(allocator), | ||
| 291 | .build_root = build_root, | ||
| 292 | .cache_root = parent.cache_root, | ||
| 293 | .global_cache_root = parent.global_cache_root, | ||
| 294 | .release_mode = parent.release_mode, | ||
| 295 | .is_release = parent.is_release, | ||
| 296 | .override_lib_dir = parent.override_lib_dir, | ||
| 297 | .debug_log_scopes = parent.debug_log_scopes, | ||
| 298 | .debug_compile_errors = parent.debug_compile_errors, | ||
| 299 | .enable_darling = parent.enable_darling, | ||
| 300 | .enable_qemu = parent.enable_qemu, | ||
| 301 | .enable_rosetta = parent.enable_rosetta, | ||
| 302 | .enable_wasmtime = parent.enable_wasmtime, | ||
| 303 | .enable_wine = parent.enable_wine, | ||
| 304 | .glibc_runtimes_dir = parent.glibc_runtimes_dir, | ||
| 305 | .host = parent.host, | ||
| 306 | .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }), | ||
| 307 | }; | ||
| 308 | try child.top_level_steps.append(&child.install_tls); | ||
| 309 | try child.top_level_steps.append(&child.uninstall_tls); | ||
| 310 | child.default_step = &child.install_tls.step; | ||
| 311 | return child; | ||
| 312 | } | ||
| 313 | |||
| 314 | pub fn applyArgs(b: *Builder, args: anytype) !void { | ||
| 315 | // TODO this function is the way that a build.zig file communicates | ||
| 316 | // options to its dependencies. It is the programmatic way to give | ||
| 317 | // command line arguments to a build.zig script. | ||
| 318 | _ = b; | ||
| 319 | _ = args; | ||
| 320 | } | ||
| 321 | |||
| 236 | pub fn destroy(self: *Builder) void { | 322 | pub fn destroy(self: *Builder) void { |
| 237 | self.env_map.deinit(); | 323 | self.env_map.deinit(); |
| 238 | self.top_level_steps.deinit(); | 324 | self.top_level_steps.deinit(); |
| ... | @@ -1300,6 +1386,70 @@ pub const Builder = struct { | ... | @@ -1300,6 +1386,70 @@ pub const Builder = struct { |
| 1300 | &[_][]const u8{ base_dir, dest_rel_path }, | 1386 | &[_][]const u8{ base_dir, dest_rel_path }, |
| 1301 | ) catch unreachable; | 1387 | ) catch unreachable; |
| 1302 | } | 1388 | } |
| 1389 | |||
| 1390 | pub const Dependency = struct { | ||
| 1391 | builder: *Builder, | ||
| 1392 | |||
| 1393 | pub fn artifact(d: *Dependency, name: []const u8) *LibExeObjStep { | ||
| 1394 | var found: ?*LibExeObjStep = null; | ||
| 1395 | for (d.builder.install_tls.step.dependencies.items) |dep_step| { | ||
| 1396 | const inst = dep_step.cast(InstallArtifactStep) orelse continue; | ||
| 1397 | if (mem.eql(u8, inst.artifact.name, name)) { | ||
| 1398 | if (found != null) panic("artifact name '{s}' is ambiguous", .{name}); | ||
| 1399 | found = inst.artifact; | ||
| 1400 | } | ||
| 1401 | } | ||
| 1402 | return found orelse { | ||
| 1403 | for (d.builder.install_tls.step.dependencies.items) |dep_step| { | ||
| 1404 | const inst = dep_step.cast(InstallArtifactStep) orelse continue; | ||
| 1405 | log.info("available artifact: '{s}'", .{inst.artifact.name}); | ||
| 1406 | } | ||
| 1407 | panic("unable to find artifact '{s}'", .{name}); | ||
| 1408 | }; | ||
| 1409 | } | ||
| 1410 | }; | ||
| 1411 | |||
| 1412 | pub fn dependency(b: *Builder, name: []const u8, args: anytype) *Dependency { | ||
| 1413 | const build_runner = @import("root"); | ||
| 1414 | const deps = build_runner.dependencies; | ||
| 1415 | |||
| 1416 | inline for (@typeInfo(deps.imports).Struct.decls) |decl| { | ||
| 1417 | if (mem.startsWith(u8, decl.name, b.dep_prefix) and | ||
| 1418 | mem.endsWith(u8, decl.name, name) and | ||
| 1419 | decl.name.len == b.dep_prefix.len + name.len) | ||
| 1420 | { | ||
| 1421 | const build_zig = @field(deps.imports, decl.name); | ||
| 1422 | const build_root = @field(deps.build_root, decl.name); | ||
| 1423 | return dependencyInner(b, name, build_root, build_zig, args); | ||
| 1424 | } | ||
| 1425 | } | ||
| 1426 | |||
| 1427 | const full_path = b.pathFromRoot("build.zig.ini"); | ||
| 1428 | std.debug.print("no dependency named '{s}' in '{s}'\n", .{ name, full_path }); | ||
| 1429 | std.process.exit(1); | ||
| 1430 | } | ||
| 1431 | |||
| 1432 | fn dependencyInner( | ||
| 1433 | b: *Builder, | ||
| 1434 | name: []const u8, | ||
| 1435 | build_root: []const u8, | ||
| 1436 | comptime build_zig: type, | ||
| 1437 | args: anytype, | ||
| 1438 | ) *Dependency { | ||
| 1439 | const sub_builder = b.createChild(name, build_root, args) catch unreachable; | ||
| 1440 | sub_builder.runBuild(build_zig) catch unreachable; | ||
| 1441 | const dep = b.allocator.create(Dependency) catch unreachable; | ||
| 1442 | dep.* = .{ .builder = sub_builder }; | ||
| 1443 | return dep; | ||
| 1444 | } | ||
| 1445 | |||
| 1446 | pub fn runBuild(b: *Builder, build_zig: anytype) anyerror!void { | ||
| 1447 | switch (@typeInfo(@typeInfo(@TypeOf(build_zig.build)).Fn.return_type.?)) { | ||
| 1448 | .Void => build_zig.build(b), | ||
| 1449 | .ErrorUnion => try build_zig.build(b), | ||
| 1450 | else => @compileError("expected return type of build to be 'void' or '!void'"), | ||
| 1451 | } | ||
| 1452 | } | ||
| 1303 | }; | 1453 | }; |
| 1304 | 1454 | ||
| 1305 | test "builder.findProgram compiles" { | 1455 | test "builder.findProgram compiles" { |
src/Package.zig+99-34
| ... | @@ -12,6 +12,8 @@ const Compilation = @import("Compilation.zig"); | ... | @@ -12,6 +12,8 @@ const Compilation = @import("Compilation.zig"); |
| 12 | const Module = @import("Module.zig"); | 12 | const Module = @import("Module.zig"); |
| 13 | const ThreadPool = @import("ThreadPool.zig"); | 13 | const ThreadPool = @import("ThreadPool.zig"); |
| 14 | const WaitGroup = @import("WaitGroup.zig"); | 14 | const WaitGroup = @import("WaitGroup.zig"); |
| 15 | const Cache = @import("Cache.zig"); | ||
| 16 | const build_options = @import("build_options"); | ||
| 15 | 17 | ||
| 16 | pub const Table = std.StringHashMapUnmanaged(*Package); | 18 | pub const Table = std.StringHashMapUnmanaged(*Package); |
| 17 | 19 | ||
| ... | @@ -139,6 +141,9 @@ pub fn fetchAndAddDependencies( | ... | @@ -139,6 +141,9 @@ pub fn fetchAndAddDependencies( |
| 139 | directory: Compilation.Directory, | 141 | directory: Compilation.Directory, |
| 140 | global_cache_directory: Compilation.Directory, | 142 | global_cache_directory: Compilation.Directory, |
| 141 | local_cache_directory: Compilation.Directory, | 143 | local_cache_directory: Compilation.Directory, |
| 144 | dependencies_source: *std.ArrayList(u8), | ||
| 145 | build_roots_source: *std.ArrayList(u8), | ||
| 146 | name_prefix: []const u8, | ||
| 142 | ) !void { | 147 | ) !void { |
| 143 | const max_bytes = 10 * 1024 * 1024; | 148 | const max_bytes = 10 * 1024 * 1024; |
| 144 | const gpa = thread_pool.allocator; | 149 | const gpa = thread_pool.allocator; |
| ... | @@ -156,15 +161,15 @@ pub fn fetchAndAddDependencies( | ... | @@ -156,15 +161,15 @@ pub fn fetchAndAddDependencies( |
| 156 | var it = ini.iterateSection("\n[dependency]\n"); | 161 | var it = ini.iterateSection("\n[dependency]\n"); |
| 157 | while (it.next()) |dep| { | 162 | while (it.next()) |dep| { |
| 158 | var line_it = mem.split(u8, dep, "\n"); | 163 | var line_it = mem.split(u8, dep, "\n"); |
| 159 | var opt_id: ?[]const u8 = null; | 164 | var opt_name: ?[]const u8 = null; |
| 160 | var opt_url: ?[]const u8 = null; | 165 | var opt_url: ?[]const u8 = null; |
| 161 | var expected_hash: ?[]const u8 = null; | 166 | var expected_hash: ?[]const u8 = null; |
| 162 | while (line_it.next()) |kv| { | 167 | while (line_it.next()) |kv| { |
| 163 | const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; | 168 | const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue; |
| 164 | const key = kv[0..eq_pos]; | 169 | const key = kv[0..eq_pos]; |
| 165 | const value = kv[eq_pos + 1 ..]; | 170 | const value = kv[eq_pos + 1 ..]; |
| 166 | if (mem.eql(u8, key, "id")) { | 171 | if (mem.eql(u8, key, "name")) { |
| 167 | opt_id = value; | 172 | opt_name = value; |
| 168 | } else if (mem.eql(u8, key, "url")) { | 173 | } else if (mem.eql(u8, key, "url")) { |
| 169 | opt_url = value; | 174 | opt_url = value; |
| 170 | } else if (mem.eql(u8, key, "hash")) { | 175 | } else if (mem.eql(u8, key, "hash")) { |
| ... | @@ -181,9 +186,9 @@ pub fn fetchAndAddDependencies( | ... | @@ -181,9 +186,9 @@ pub fn fetchAndAddDependencies( |
| 181 | } | 186 | } |
| 182 | } | 187 | } |
| 183 | 188 | ||
| 184 | const id = opt_id orelse { | 189 | const name = opt_name orelse { |
| 185 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); | 190 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); |
| 186 | std.log.err("{s}/{s}:{d}:{d} missing key: 'id'", .{ | 191 | std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{ |
| 187 | directory.path orelse ".", | 192 | directory.path orelse ".", |
| 188 | "build.zig.ini", | 193 | "build.zig.ini", |
| 189 | loc.line, | 194 | loc.line, |
| ... | @@ -195,7 +200,7 @@ pub fn fetchAndAddDependencies( | ... | @@ -195,7 +200,7 @@ pub fn fetchAndAddDependencies( |
| 195 | 200 | ||
| 196 | const url = opt_url orelse { | 201 | const url = opt_url orelse { |
| 197 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); | 202 | const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr)); |
| 198 | std.log.err("{s}/{s}:{d}:{d} missing key: 'id'", .{ | 203 | std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{ |
| 199 | directory.path orelse ".", | 204 | directory.path orelse ".", |
| 200 | "build.zig.ini", | 205 | "build.zig.ini", |
| 201 | loc.line, | 206 | loc.line, |
| ... | @@ -205,6 +210,10 @@ pub fn fetchAndAddDependencies( | ... | @@ -205,6 +210,10 @@ pub fn fetchAndAddDependencies( |
| 205 | continue; | 210 | continue; |
| 206 | }; | 211 | }; |
| 207 | 212 | ||
| 213 | const sub_prefix = try std.fmt.allocPrint(gpa, "{s}{s}.", .{ name_prefix, name }); | ||
| 214 | defer gpa.free(sub_prefix); | ||
| 215 | const fqn = sub_prefix[0 .. sub_prefix.len - 1]; | ||
| 216 | |||
| 208 | const sub_pkg = try fetchAndUnpack( | 217 | const sub_pkg = try fetchAndUnpack( |
| 209 | thread_pool, | 218 | thread_pool, |
| 210 | http_client, | 219 | http_client, |
| ... | @@ -213,22 +222,56 @@ pub fn fetchAndAddDependencies( | ... | @@ -213,22 +222,56 @@ pub fn fetchAndAddDependencies( |
| 213 | expected_hash, | 222 | expected_hash, |
| 214 | ini, | 223 | ini, |
| 215 | directory, | 224 | directory, |
| 225 | build_roots_source, | ||
| 226 | fqn, | ||
| 216 | ); | 227 | ); |
| 217 | 228 | ||
| 218 | try sub_pkg.fetchAndAddDependencies( | 229 | try pkg.fetchAndAddDependencies( |
| 219 | thread_pool, | 230 | thread_pool, |
| 220 | http_client, | 231 | http_client, |
| 221 | sub_pkg.root_src_directory, | 232 | sub_pkg.root_src_directory, |
| 222 | global_cache_directory, | 233 | global_cache_directory, |
| 223 | local_cache_directory, | 234 | local_cache_directory, |
| 235 | dependencies_source, | ||
| 236 | build_roots_source, | ||
| 237 | sub_prefix, | ||
| 224 | ); | 238 | ); |
| 225 | 239 | ||
| 226 | try addAndAdopt(pkg, gpa, id, sub_pkg); | 240 | try addAndAdopt(pkg, gpa, fqn, sub_pkg); |
| 241 | |||
| 242 | try dependencies_source.writer().print(" pub const {s} = @import(\"{}\");\n", .{ | ||
| 243 | std.zig.fmtId(fqn), std.zig.fmtEscapes(fqn), | ||
| 244 | }); | ||
| 227 | } | 245 | } |
| 228 | 246 | ||
| 229 | if (any_error) return error.InvalidBuildZigIniFile; | 247 | if (any_error) return error.InvalidBuildZigIniFile; |
| 230 | } | 248 | } |
| 231 | 249 | ||
| 250 | pub fn createFilePkg( | ||
| 251 | gpa: Allocator, | ||
| 252 | global_cache_directory: Compilation.Directory, | ||
| 253 | basename: []const u8, | ||
| 254 | contents: []const u8, | ||
| 255 | ) !*Package { | ||
| 256 | const rand_int = std.crypto.random.int(u64); | ||
| 257 | const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ hex64(rand_int); | ||
| 258 | { | ||
| 259 | var tmp_dir = try global_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); | ||
| 260 | defer tmp_dir.close(); | ||
| 261 | try tmp_dir.writeFile(basename, contents); | ||
| 262 | } | ||
| 263 | |||
| 264 | var hh: Cache.HashHelper = .{}; | ||
| 265 | hh.addBytes(build_options.version); | ||
| 266 | hh.addBytes(contents); | ||
| 267 | const hex_digest = hh.final(); | ||
| 268 | |||
| 269 | const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; | ||
| 270 | try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); | ||
| 271 | |||
| 272 | return createWithDir(gpa, global_cache_directory, o_dir_sub_path, basename); | ||
| 273 | } | ||
| 274 | |||
| 232 | fn fetchAndUnpack( | 275 | fn fetchAndUnpack( |
| 233 | thread_pool: *ThreadPool, | 276 | thread_pool: *ThreadPool, |
| 234 | http_client: *std.http.Client, | 277 | http_client: *std.http.Client, |
| ... | @@ -237,6 +280,8 @@ fn fetchAndUnpack( | ... | @@ -237,6 +280,8 @@ fn fetchAndUnpack( |
| 237 | expected_hash: ?[]const u8, | 280 | expected_hash: ?[]const u8, |
| 238 | ini: std.Ini, | 281 | ini: std.Ini, |
| 239 | comp_directory: Compilation.Directory, | 282 | comp_directory: Compilation.Directory, |
| 283 | build_roots_source: *std.ArrayList(u8), | ||
| 284 | fqn: []const u8, | ||
| 240 | ) !*Package { | 285 | ) !*Package { |
| 241 | const gpa = http_client.allocator; | 286 | const gpa = http_client.allocator; |
| 242 | const s = fs.path.sep_str; | 287 | const s = fs.path.sep_str; |
| ... | @@ -267,14 +312,22 @@ fn fetchAndUnpack( | ... | @@ -267,14 +312,22 @@ fn fetchAndUnpack( |
| 267 | const owned_src_path = try gpa.dupe(u8, build_zig_basename); | 312 | const owned_src_path = try gpa.dupe(u8, build_zig_basename); |
| 268 | errdefer gpa.free(owned_src_path); | 313 | errdefer gpa.free(owned_src_path); |
| 269 | 314 | ||
| 315 | const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); | ||
| 316 | errdefer gpa.free(build_root); | ||
| 317 | |||
| 318 | try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{ | ||
| 319 | std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root), | ||
| 320 | }); | ||
| 321 | |||
| 270 | ptr.* = .{ | 322 | ptr.* = .{ |
| 271 | .root_src_directory = .{ | 323 | .root_src_directory = .{ |
| 272 | .path = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}), | 324 | .path = build_root, |
| 273 | .handle = pkg_dir, | 325 | .handle = pkg_dir, |
| 274 | }, | 326 | }, |
| 275 | .root_src_directory_owned = true, | 327 | .root_src_directory_owned = true, |
| 276 | .root_src_path = owned_src_path, | 328 | .root_src_path = owned_src_path, |
| 277 | }; | 329 | }; |
| 330 | |||
| 278 | return ptr; | 331 | return ptr; |
| 279 | } | 332 | } |
| 280 | 333 | ||
| ... | @@ -331,31 +384,7 @@ fn fetchAndUnpack( | ... | @@ -331,31 +384,7 @@ fn fetchAndUnpack( |
| 331 | }; | 384 | }; |
| 332 | 385 | ||
| 333 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); | 386 | const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash); |
| 334 | 387 | try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, pkg_dir_sub_path); | |
| 335 | { | ||
| 336 | // Rename the temporary directory into the global package cache. | ||
| 337 | var handled_missing_dir = false; | ||
| 338 | while (true) { | ||
| 339 | global_cache_directory.handle.rename(tmp_dir_sub_path, pkg_dir_sub_path) catch |err| switch (err) { | ||
| 340 | error.FileNotFound => { | ||
| 341 | if (handled_missing_dir) return err; | ||
| 342 | global_cache_directory.handle.makeDir("p") catch |mkd_err| switch (mkd_err) { | ||
| 343 | error.PathAlreadyExists => handled_missing_dir = true, | ||
| 344 | else => |e| return e, | ||
| 345 | }; | ||
| 346 | continue; | ||
| 347 | }, | ||
| 348 | error.PathAlreadyExists => { | ||
| 349 | // Package has been already downloaded and may already be in use on the system. | ||
| 350 | global_cache_directory.handle.deleteTree(tmp_dir_sub_path) catch |del_err| { | ||
| 351 | std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); | ||
| 352 | }; | ||
| 353 | }, | ||
| 354 | else => |e| return e, | ||
| 355 | }; | ||
| 356 | break; | ||
| 357 | } | ||
| 358 | } | ||
| 359 | 388 | ||
| 360 | if (expected_hash) |h| { | 389 | if (expected_hash) |h| { |
| 361 | const actual_hex = hexDigest(actual_hash); | 390 | const actual_hex = hexDigest(actual_hash); |
| ... | @@ -378,6 +407,13 @@ fn fetchAndUnpack( | ... | @@ -378,6 +407,13 @@ fn fetchAndUnpack( |
| 378 | ); | 407 | ); |
| 379 | } | 408 | } |
| 380 | 409 | ||
| 410 | const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path}); | ||
| 411 | defer gpa.free(build_root); | ||
| 412 | |||
| 413 | try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{ | ||
| 414 | std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root), | ||
| 415 | }); | ||
| 416 | |||
| 381 | return createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename); | 417 | return createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename); |
| 382 | } | 418 | } |
| 383 | 419 | ||
| ... | @@ -516,3 +552,32 @@ fn hexDigest(digest: [Hash.digest_length]u8) [Hash.digest_length * 2]u8 { | ... | @@ -516,3 +552,32 @@ fn hexDigest(digest: [Hash.digest_length]u8) [Hash.digest_length * 2]u8 { |
| 516 | } | 552 | } |
| 517 | return result; | 553 | return result; |
| 518 | } | 554 | } |
| 555 | |||
| 556 | fn renameTmpIntoCache( | ||
| 557 | cache_dir: fs.Dir, | ||
| 558 | tmp_dir_sub_path: []const u8, | ||
| 559 | dest_dir_sub_path: []const u8, | ||
| 560 | ) !void { | ||
| 561 | assert(dest_dir_sub_path[1] == '/'); | ||
| 562 | var handled_missing_dir = false; | ||
| 563 | while (true) { | ||
| 564 | cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) { | ||
| 565 | error.FileNotFound => { | ||
| 566 | if (handled_missing_dir) return err; | ||
| 567 | cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) { | ||
| 568 | error.PathAlreadyExists => handled_missing_dir = true, | ||
| 569 | else => |e| return e, | ||
| 570 | }; | ||
| 571 | continue; | ||
| 572 | }, | ||
| 573 | error.PathAlreadyExists => { | ||
| 574 | // Package has been already downloaded and may already be in use on the system. | ||
| 575 | cache_dir.deleteTree(tmp_dir_sub_path) catch |del_err| { | ||
| 576 | std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)}); | ||
| 577 | }; | ||
| 578 | }, | ||
| 579 | else => |e| return e, | ||
| 580 | }; | ||
| 581 | break; | ||
| 582 | } | ||
| 583 | } |
src/main.zig+42-11
| ... | @@ -3983,11 +3983,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi | ... | @@ -3983,11 +3983,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi |
| 3983 | }; | 3983 | }; |
| 3984 | defer zig_lib_directory.handle.close(); | 3984 | defer zig_lib_directory.handle.close(); |
| 3985 | 3985 | ||
| 3986 | var main_pkg: Package = .{ | ||
| 3987 | .root_src_directory = zig_lib_directory, | ||
| 3988 | .root_src_path = "build_runner.zig", | ||
| 3989 | }; | ||
| 3990 | |||
| 3991 | var cleanup_build_dir: ?fs.Dir = null; | 3986 | var cleanup_build_dir: ?fs.Dir = null; |
| 3992 | defer if (cleanup_build_dir) |*dir| dir.close(); | 3987 | defer if (cleanup_build_dir) |*dir| dir.close(); |
| 3993 | 3988 | ||
| ... | @@ -4031,12 +4026,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi | ... | @@ -4031,12 +4026,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi |
| 4031 | }; | 4026 | }; |
| 4032 | child_argv.items[argv_index_build_file] = build_directory.path orelse cwd_path; | 4027 | child_argv.items[argv_index_build_file] = build_directory.path orelse cwd_path; |
| 4033 | 4028 | ||
| 4034 | var build_pkg: Package = .{ | ||
| 4035 | .root_src_directory = build_directory, | ||
| 4036 | .root_src_path = build_zig_basename, | ||
| 4037 | }; | ||
| 4038 | try main_pkg.addAndAdopt(arena, "@build", &build_pkg); | ||
| 4039 | |||
| 4040 | var global_cache_directory: Compilation.Directory = l: { | 4029 | var global_cache_directory: Compilation.Directory = l: { |
| 4041 | const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); | 4030 | const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); |
| 4042 | break :l .{ | 4031 | break :l .{ |
| ... | @@ -4083,23 +4072,65 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi | ... | @@ -4083,23 +4072,65 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi |
| 4083 | try thread_pool.init(gpa); | 4072 | try thread_pool.init(gpa); |
| 4084 | defer thread_pool.deinit(); | 4073 | defer thread_pool.deinit(); |
| 4085 | 4074 | ||
| 4075 | var main_pkg: Package = .{ | ||
| 4076 | .root_src_directory = zig_lib_directory, | ||
| 4077 | .root_src_path = "build_runner.zig", | ||
| 4078 | }; | ||
| 4079 | |||
| 4086 | if (!build_options.omit_pkg_fetching_code) { | 4080 | if (!build_options.omit_pkg_fetching_code) { |
| 4087 | var http_client: std.http.Client = .{ .allocator = gpa }; | 4081 | var http_client: std.http.Client = .{ .allocator = gpa }; |
| 4088 | defer http_client.deinit(); | 4082 | defer http_client.deinit(); |
| 4089 | try http_client.rescanRootCertificates(); | 4083 | try http_client.rescanRootCertificates(); |
| 4090 | 4084 | ||
| 4085 | // Here we provide an import to the build runner that allows using reflection to find | ||
| 4086 | // all of the dependencies. Without this, there would be no way to use `@import` to | ||
| 4087 | // access dependencies by name, since `@import` requires string literals. | ||
| 4088 | var dependencies_source = std.ArrayList(u8).init(gpa); | ||
| 4089 | defer dependencies_source.deinit(); | ||
| 4090 | try dependencies_source.appendSlice("pub const imports = struct {\n"); | ||
| 4091 | |||
| 4092 | // This will go into the same package. It contains the file system paths | ||
| 4093 | // to all the build.zig files. | ||
| 4094 | var build_roots_source = std.ArrayList(u8).init(gpa); | ||
| 4095 | defer build_roots_source.deinit(); | ||
| 4096 | |||
| 4097 | // Here we borrow main package's table and will replace it with a fresh | ||
| 4098 | // one after this process completes. | ||
| 4091 | main_pkg.fetchAndAddDependencies( | 4099 | main_pkg.fetchAndAddDependencies( |
| 4092 | &thread_pool, | 4100 | &thread_pool, |
| 4093 | &http_client, | 4101 | &http_client, |
| 4094 | build_directory, | 4102 | build_directory, |
| 4095 | global_cache_directory, | 4103 | global_cache_directory, |
| 4096 | local_cache_directory, | 4104 | local_cache_directory, |
| 4105 | &dependencies_source, | ||
| 4106 | &build_roots_source, | ||
| 4107 | "", | ||
| 4097 | ) catch |err| switch (err) { | 4108 | ) catch |err| switch (err) { |
| 4098 | error.PackageFetchFailed => process.exit(1), | 4109 | error.PackageFetchFailed => process.exit(1), |
| 4099 | else => |e| return e, | 4110 | else => |e| return e, |
| 4100 | }; | 4111 | }; |
| 4112 | |||
| 4113 | try dependencies_source.appendSlice("};\npub const build_root = struct {\n"); | ||
| 4114 | try dependencies_source.appendSlice(build_roots_source.items); | ||
| 4115 | try dependencies_source.appendSlice("};\n"); | ||
| 4116 | |||
| 4117 | const deps_pkg = try Package.createFilePkg( | ||
| 4118 | gpa, | ||
| 4119 | global_cache_directory, | ||
| 4120 | "dependencies.zig", | ||
| 4121 | dependencies_source.items, | ||
| 4122 | ); | ||
| 4123 | |||
| 4124 | mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table); | ||
| 4125 | try main_pkg.addAndAdopt(gpa, "@dependencies", deps_pkg); | ||
| 4101 | } | 4126 | } |
| 4102 | 4127 | ||
| 4128 | var build_pkg: Package = .{ | ||
| 4129 | .root_src_directory = build_directory, | ||
| 4130 | .root_src_path = build_zig_basename, | ||
| 4131 | }; | ||
| 4132 | try main_pkg.addAndAdopt(gpa, "@build", &build_pkg); | ||
| 4133 | |||
| 4103 | const comp = Compilation.create(gpa, .{ | 4134 | const comp = Compilation.create(gpa, .{ |
| 4104 | .zig_lib_directory = zig_lib_directory, | 4135 | .zig_lib_directory = zig_lib_directory, |
| 4105 | .local_cache_directory = local_cache_directory, | 4136 | .local_cache_directory = local_cache_directory, |