authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-12 18:49:15-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-01-12 18:49:15-05:00
log7cb2f9222da38d687e8708dd5d94d3175cc77995
treeae6a7cf313236d085999c2a99fd13241704f3c74
parentcbbf8c8a2d77d84ce88ea1cef9a3e7d54081e33d
parentf4d6b37068db7ef3b5828dbe2403e65bf64a0f2c
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #14265 from ziglang/init-package-manager

Package Manager MVP

16 files changed, 1124 insertions(+), 43 deletions(-)

build.zig+4-3
......@@ -185,6 +185,7 @@ pub fn build(b: *Builder) !void {
185185 exe_options.addOption(bool, "llvm_has_arc", llvm_has_arc);
186186 exe_options.addOption(bool, "force_gpa", force_gpa);
187187 exe_options.addOption(bool, "only_c", only_c);
188 exe_options.addOption(bool, "omit_pkg_fetching_code", false);
188189
189190 if (link_libc) {
190191 exe.linkLibC();
......@@ -567,14 +568,14 @@ fn addCmakeCfgOptionsToExe(
567568 // back to -lc++ and cross our fingers.
568569 addCxxKnownPath(b, cfg, exe, b.fmt("libstdc++.{s}", .{lib_suffix}), "", need_cpp_includes) catch |err| switch (err) {
569570 error.RequiredLibraryNotFound => {
570 exe.linkSystemLibrary("c++");
571 exe.linkLibCpp();
571572 },
572573 else => |e| return e,
573574 };
574575 exe.linkSystemLibrary("unwind");
575576 },
576 .ios, .macos, .watchos, .tvos => {
577 exe.linkSystemLibrary("c++");
577 .ios, .macos, .watchos, .tvos, .windows => {
578 exe.linkLibCpp();
578579 },
579580 .freebsd => {
580581 if (static) {
lib/build_runner.zig+4-10
......@@ -9,6 +9,8 @@ const process = std.process;
99const ArrayList = std.ArrayList;
1010const File = std.fs.File;
1111
12pub const dependencies = @import("@dependencies");
13
1214pub fn main() !void {
1315 // Here we use an ArenaAllocator backed by a DirectAllocator because a build is a short-lived,
1416 // one shot program. We don't need to waste time freeing memory and finding places to squish
......@@ -207,7 +209,7 @@ pub fn main() !void {
207209
208210 builder.debug_log_scopes = debug_log_scopes.items;
209211 builder.resolveInstallPrefix(install_prefix, dir_list);
210 try runBuild(builder);
212 try builder.runBuild(root);
211213
212214 if (builder.validateUserInputDidItFail())
213215 return usageAndErr(builder, true, stderr_stream);
......@@ -223,19 +225,11 @@ pub fn main() !void {
223225 };
224226}
225227
226fn runBuild(builder: *Builder) anyerror!void {
227 switch (@typeInfo(@typeInfo(@TypeOf(root.build)).Fn.return_type.?)) {
228 .Void => root.build(builder),
229 .ErrorUnion => try root.build(builder),
230 else => @compileError("expected return type of build to be 'void' or '!void'"),
231 }
232}
233
234228fn usage(builder: *Builder, already_ran_build: bool, out_stream: anytype) !void {
235229 // run the build script to collect the options
236230 if (!already_ran_build) {
237231 builder.resolveInstallPrefix(null, .{});
238 try runBuild(builder);
232 try builder.runBuild(root);
239233 }
240234
241235 try out_stream.print(
lib/std/Ini.zig created+66
......@@ -0,0 +1,66 @@
1bytes: []const u8,
2
3pub const SectionIterator = struct {
4 ini: Ini,
5 next_index: ?usize,
6 header: []const u8,
7
8 pub fn next(it: *SectionIterator) ?[]const u8 {
9 const bytes = it.ini.bytes;
10 const start = it.next_index orelse return null;
11 const end = mem.indexOfPos(u8, bytes, start, "\n[") orelse bytes.len;
12 const result = bytes[start..end];
13 if (mem.indexOfPos(u8, bytes, start, it.header)) |next_index| {
14 it.next_index = next_index + it.header.len;
15 } else {
16 it.next_index = null;
17 }
18 return result;
19 }
20};
21
22/// Asserts that `header` includes "\n[" at the beginning and "]\n" at the end.
23/// `header` must remain valid for the lifetime of the iterator.
24pub fn iterateSection(ini: Ini, header: []const u8) SectionIterator {
25 assert(mem.startsWith(u8, header, "\n["));
26 assert(mem.endsWith(u8, header, "]\n"));
27 const first_header = header[1..];
28 const next_index = if (mem.indexOf(u8, ini.bytes, first_header)) |i|
29 i + first_header.len
30 else
31 null;
32 return .{
33 .ini = ini,
34 .next_index = next_index,
35 .header = header,
36 };
37}
38
39const std = @import("std.zig");
40const mem = std.mem;
41const assert = std.debug.assert;
42const Ini = @This();
43const testing = std.testing;
44
45test iterateSection {
46 const example =
47 \\[package]
48 \\name=libffmpeg
49 \\version=5.1.2
50 \\
51 \\[dependency]
52 \\id=libz
53 \\url=url1
54 \\
55 \\[dependency]
56 \\id=libmp3lame
57 \\url=url2
58 ;
59 var ini: Ini = .{ .bytes = example };
60 var it = ini.iterateSection("\n[dependency]\n");
61 const section1 = it.next() orelse return error.TestFailed;
62 try testing.expectEqualStrings("id=libz\nurl=url1\n", section1);
63 const section2 = it.next() orelse return error.TestFailed;
64 try testing.expectEqualStrings("id=libmp3lame\nurl=url2", section2);
65 try testing.expect(it.next() == null);
66}
lib/std/build.zig+159-2
......@@ -69,13 +69,15 @@ pub const Builder = struct {
6969 search_prefixes: ArrayList([]const u8),
7070 libc_file: ?[]const u8 = null,
7171 installed_files: ArrayList(InstalledFile),
72 /// Path to the directory containing build.zig.
7273 build_root: []const u8,
7374 cache_root: []const u8,
7475 global_cache_root: []const u8,
7576 release_mode: ?std.builtin.Mode,
7677 is_release: bool,
78 /// zig lib dir
7779 override_lib_dir: ?[]const u8,
78 vcpkg_root: VcpkgRoot,
80 vcpkg_root: VcpkgRoot = .unattempted,
7981 pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
8082 args: ?[][]const u8 = null,
8183 debug_log_scopes: []const []const u8 = &.{},
......@@ -100,6 +102,8 @@ pub const Builder = struct {
100102 /// Information about the native target. Computed before build() is invoked.
101103 host: NativeTargetInfo,
102104
105 dep_prefix: []const u8 = "",
106
103107 pub const ExecError = error{
104108 ReadFailure,
105109 ExitCodeFailure,
......@@ -223,7 +227,6 @@ pub const Builder = struct {
223227 .is_release = false,
224228 .override_lib_dir = null,
225229 .install_path = undefined,
226 .vcpkg_root = VcpkgRoot{ .unattempted = {} },
227230 .args = null,
228231 .host = host,
229232 };
......@@ -233,6 +236,92 @@ pub const Builder = struct {
233236 return self;
234237 }
235238
239 fn createChild(
240 parent: *Builder,
241 dep_name: []const u8,
242 build_root: []const u8,
243 args: anytype,
244 ) !*Builder {
245 const child = try createChildOnly(parent, dep_name, build_root);
246 try applyArgs(child, args);
247 return child;
248 }
249
250 fn createChildOnly(parent: *Builder, dep_name: []const u8, build_root: []const u8) !*Builder {
251 const allocator = parent.allocator;
252 const child = try allocator.create(Builder);
253 child.* = .{
254 .allocator = allocator,
255 .install_tls = .{
256 .step = Step.initNoOp(.top_level, "install", allocator),
257 .description = "Copy build artifacts to prefix path",
258 },
259 .uninstall_tls = .{
260 .step = Step.init(.top_level, "uninstall", allocator, makeUninstall),
261 .description = "Remove build artifacts from prefix path",
262 },
263 .user_input_options = UserInputOptionsMap.init(allocator),
264 .available_options_map = AvailableOptionsMap.init(allocator),
265 .available_options_list = ArrayList(AvailableOption).init(allocator),
266 .verbose = parent.verbose,
267 .verbose_link = parent.verbose_link,
268 .verbose_cc = parent.verbose_cc,
269 .verbose_air = parent.verbose_air,
270 .verbose_llvm_ir = parent.verbose_llvm_ir,
271 .verbose_cimport = parent.verbose_cimport,
272 .verbose_llvm_cpu_features = parent.verbose_llvm_cpu_features,
273 .prominent_compile_errors = parent.prominent_compile_errors,
274 .color = parent.color,
275 .reference_trace = parent.reference_trace,
276 .invalid_user_input = false,
277 .zig_exe = parent.zig_exe,
278 .default_step = undefined,
279 .env_map = parent.env_map,
280 .top_level_steps = ArrayList(*TopLevelStep).init(allocator),
281 .install_prefix = undefined,
282 .dest_dir = parent.dest_dir,
283 .lib_dir = parent.lib_dir,
284 .exe_dir = parent.exe_dir,
285 .h_dir = parent.h_dir,
286 .install_path = parent.install_path,
287 .sysroot = parent.sysroot,
288 .search_prefixes = ArrayList([]const u8).init(allocator),
289 .libc_file = parent.libc_file,
290 .installed_files = ArrayList(InstalledFile).init(allocator),
291 .build_root = build_root,
292 .cache_root = parent.cache_root,
293 .global_cache_root = parent.global_cache_root,
294 .release_mode = parent.release_mode,
295 .is_release = parent.is_release,
296 .override_lib_dir = parent.override_lib_dir,
297 .debug_log_scopes = parent.debug_log_scopes,
298 .debug_compile_errors = parent.debug_compile_errors,
299 .enable_darling = parent.enable_darling,
300 .enable_qemu = parent.enable_qemu,
301 .enable_rosetta = parent.enable_rosetta,
302 .enable_wasmtime = parent.enable_wasmtime,
303 .enable_wine = parent.enable_wine,
304 .glibc_runtimes_dir = parent.glibc_runtimes_dir,
305 .host = parent.host,
306 .dep_prefix = parent.fmt("{s}{s}.", .{ parent.dep_prefix, dep_name }),
307 };
308 try child.top_level_steps.append(&child.install_tls);
309 try child.top_level_steps.append(&child.uninstall_tls);
310 child.default_step = &child.install_tls.step;
311 return child;
312 }
313
314 fn applyArgs(b: *Builder, args: anytype) !void {
315 // TODO this function is the way that a build.zig file communicates
316 // options to its dependencies. It is the programmatic way to give
317 // command line arguments to a build.zig script.
318 _ = args;
319 // TODO create a hash based on the args and the package hash, use this
320 // to compute the install prefix.
321 const install_prefix = b.pathJoin(&.{ b.cache_root, "pkg" });
322 b.resolveInstallPrefix(install_prefix, .{});
323 }
324
236325 pub fn destroy(self: *Builder) void {
237326 self.env_map.deinit();
238327 self.top_level_steps.deinit();
......@@ -1068,6 +1157,10 @@ pub const Builder = struct {
10681157 return self.addInstallFileWithDir(source.dupe(self), .lib, dest_rel_path);
10691158 }
10701159
1160 pub fn addInstallHeaderFile(b: *Builder, src_path: []const u8, dest_rel_path: []const u8) *InstallFileStep {
1161 return b.addInstallFileWithDir(.{ .path = src_path }, .header, dest_rel_path);
1162 }
1163
10711164 pub fn addInstallRaw(self: *Builder, artifact: *LibExeObjStep, dest_filename: []const u8, options: InstallRawStep.CreateOptions) *InstallRawStep {
10721165 return InstallRawStep.create(self, artifact, dest_filename, options);
10731166 }
......@@ -1300,6 +1393,70 @@ pub const Builder = struct {
13001393 &[_][]const u8{ base_dir, dest_rel_path },
13011394 ) catch unreachable;
13021395 }
1396
1397 pub const Dependency = struct {
1398 builder: *Builder,
1399
1400 pub fn artifact(d: *Dependency, name: []const u8) *LibExeObjStep {
1401 var found: ?*LibExeObjStep = null;
1402 for (d.builder.install_tls.step.dependencies.items) |dep_step| {
1403 const inst = dep_step.cast(InstallArtifactStep) orelse continue;
1404 if (mem.eql(u8, inst.artifact.name, name)) {
1405 if (found != null) panic("artifact name '{s}' is ambiguous", .{name});
1406 found = inst.artifact;
1407 }
1408 }
1409 return found orelse {
1410 for (d.builder.install_tls.step.dependencies.items) |dep_step| {
1411 const inst = dep_step.cast(InstallArtifactStep) orelse continue;
1412 log.info("available artifact: '{s}'", .{inst.artifact.name});
1413 }
1414 panic("unable to find artifact '{s}'", .{name});
1415 };
1416 }
1417 };
1418
1419 pub fn dependency(b: *Builder, name: []const u8, args: anytype) *Dependency {
1420 const build_runner = @import("root");
1421 const deps = build_runner.dependencies;
1422
1423 inline for (@typeInfo(deps.imports).Struct.decls) |decl| {
1424 if (mem.startsWith(u8, decl.name, b.dep_prefix) and
1425 mem.endsWith(u8, decl.name, name) and
1426 decl.name.len == b.dep_prefix.len + name.len)
1427 {
1428 const build_zig = @field(deps.imports, decl.name);
1429 const build_root = @field(deps.build_root, decl.name);
1430 return dependencyInner(b, name, build_root, build_zig, args);
1431 }
1432 }
1433
1434 const full_path = b.pathFromRoot("build.zig.ini");
1435 std.debug.print("no dependency named '{s}' in '{s}'\n", .{ name, full_path });
1436 std.process.exit(1);
1437 }
1438
1439 fn dependencyInner(
1440 b: *Builder,
1441 name: []const u8,
1442 build_root: []const u8,
1443 comptime build_zig: type,
1444 args: anytype,
1445 ) *Dependency {
1446 const sub_builder = b.createChild(name, build_root, args) catch unreachable;
1447 sub_builder.runBuild(build_zig) catch unreachable;
1448 const dep = b.allocator.create(Dependency) catch unreachable;
1449 dep.* = .{ .builder = sub_builder };
1450 return dep;
1451 }
1452
1453 pub fn runBuild(b: *Builder, build_zig: anytype) anyerror!void {
1454 switch (@typeInfo(@typeInfo(@TypeOf(build_zig.build)).Fn.return_type.?)) {
1455 .Void => build_zig.build(b),
1456 .ErrorUnion => try build_zig.build(b),
1457 else => @compileError("expected return type of build to be 'void' or '!void'"),
1458 }
1459 }
13031460};
13041461
13051462test "builder.findProgram compiles" {
lib/std/build/LibExeObjStep.zig+43-9
......@@ -108,6 +108,7 @@ object_src: []const u8,
108108link_objects: ArrayList(LinkObject),
109109include_dirs: ArrayList(IncludeDir),
110110c_macros: ArrayList([]const u8),
111installed_headers: ArrayList(*std.build.Step),
111112output_dir: ?[]const u8,
112113is_linking_libc: bool = false,
113114is_linking_libcpp: bool = false,
......@@ -370,6 +371,7 @@ fn initExtraArgs(
370371 .lib_paths = ArrayList([]const u8).init(builder.allocator),
371372 .rpaths = ArrayList([]const u8).init(builder.allocator),
372373 .framework_dirs = ArrayList([]const u8).init(builder.allocator),
374 .installed_headers = ArrayList(*std.build.Step).init(builder.allocator),
373375 .object_src = undefined,
374376 .c_std = Builder.CStd.C99,
375377 .override_lib_dir = null,
......@@ -472,6 +474,27 @@ pub fn installRaw(self: *LibExeObjStep, dest_filename: []const u8, options: Inst
472474 return self.builder.installRaw(self, dest_filename, options);
473475}
474476
477pub fn installHeader(a: *LibExeObjStep, src_path: []const u8) void {
478 const basename = fs.path.basename(src_path);
479 const install_file = a.builder.addInstallHeaderFile(src_path, basename);
480 a.builder.getInstallStep().dependOn(&install_file.step);
481 a.installed_headers.append(&install_file.step) catch unreachable;
482}
483
484pub fn installHeadersDirectory(
485 a: *LibExeObjStep,
486 src_dir_path: []const u8,
487 dest_rel_path: []const u8,
488) void {
489 const install_dir = a.builder.addInstallDirectory(.{
490 .source_dir = src_dir_path,
491 .install_dir = .header,
492 .install_subdir = dest_rel_path,
493 });
494 a.builder.getInstallStep().dependOn(&install_dir.step);
495 a.installed_headers.append(&install_dir.step) catch unreachable;
496}
497
475498/// Creates a `RunStep` with an executable built with `addExecutable`.
476499/// Add command line arguments with `addArg`.
477500pub fn run(exe: *LibExeObjStep) *RunStep {
......@@ -1362,7 +1385,7 @@ fn make(step: *Step) !void {
13621385
13631386 if (self.libc_file) |libc_file| {
13641387 try zig_args.append("--libc");
1365 try zig_args.append(libc_file.getPath(self.builder));
1388 try zig_args.append(libc_file.getPath(builder));
13661389 } else if (builder.libc_file) |libc_file| {
13671390 try zig_args.append("--libc");
13681391 try zig_args.append(libc_file);
......@@ -1577,7 +1600,7 @@ fn make(step: *Step) !void {
15771600 } else {
15781601 const need_cross_glibc = self.target.isGnuLibC() and self.is_linking_libc;
15791602
1580 switch (self.builder.host.getExternalExecutor(self.target_info, .{
1603 switch (builder.host.getExternalExecutor(self.target_info, .{
15811604 .qemu_fixes_dl = need_cross_glibc and builder.glibc_runtimes_dir != null,
15821605 .link_libc = self.is_linking_libc,
15831606 })) {
......@@ -1661,7 +1684,7 @@ fn make(step: *Step) !void {
16611684 switch (include_dir) {
16621685 .raw_path => |include_path| {
16631686 try zig_args.append("-I");
1664 try zig_args.append(self.builder.pathFromRoot(include_path));
1687 try zig_args.append(builder.pathFromRoot(include_path));
16651688 },
16661689 .raw_path_system => |include_path| {
16671690 if (builder.sysroot != null) {
......@@ -1670,7 +1693,7 @@ fn make(step: *Step) !void {
16701693 try zig_args.append("-isystem");
16711694 }
16721695
1673 const resolved_include_path = self.builder.pathFromRoot(include_path);
1696 const resolved_include_path = builder.pathFromRoot(include_path);
16741697
16751698 const common_include_path = if (builtin.os.tag == .windows and builder.sysroot != null and fs.path.isAbsolute(resolved_include_path)) blk: {
16761699 // We need to check for disk designator and strip it out from dir path so
......@@ -1686,10 +1709,21 @@ fn make(step: *Step) !void {
16861709
16871710 try zig_args.append(common_include_path);
16881711 },
1689 .other_step => |other| if (other.emit_h) {
1690 const h_path = other.getOutputHSource().getPath(self.builder);
1691 try zig_args.append("-isystem");
1692 try zig_args.append(fs.path.dirname(h_path).?);
1712 .other_step => |other| {
1713 if (other.emit_h) {
1714 const h_path = other.getOutputHSource().getPath(builder);
1715 try zig_args.append("-isystem");
1716 try zig_args.append(fs.path.dirname(h_path).?);
1717 }
1718 if (other.installed_headers.items.len > 0) {
1719 for (other.installed_headers.items) |install_step| {
1720 try install_step.make();
1721 }
1722 try zig_args.append("-I");
1723 try zig_args.append(builder.pathJoin(&.{
1724 other.builder.install_prefix, "include",
1725 }));
1726 }
16931727 },
16941728 .config_header_step => |config_header| {
16951729 try zig_args.append("-I");
......@@ -1790,7 +1824,7 @@ fn make(step: *Step) !void {
17901824 if (self.override_lib_dir) |dir| {
17911825 try zig_args.append("--zig-lib-dir");
17921826 try zig_args.append(builder.pathFromRoot(dir));
1793 } else if (self.builder.override_lib_dir) |dir| {
1827 } else if (builder.override_lib_dir) |dir| {
17941828 try zig_args.append("--zig-lib-dir");
17951829 try zig_args.append(builder.pathFromRoot(dir));
17961830 }
lib/std/compress/gzip.zig+3
......@@ -17,6 +17,9 @@ const FCOMMENT = 1 << 4;
1717
1818const max_string_len = 1024;
1919
20/// TODO: the fully qualified namespace to this declaration is
21/// std.compress.gzip.GzipStream which has a redundant "gzip" in the name.
22/// Instead, it should be `std.compress.gzip.Stream`.
2023pub fn GzipStream(comptime ReaderType: type) type {
2124 return struct {
2225 const Self = @This();
lib/std/http/Client.zig+131-4
......@@ -524,11 +524,133 @@ pub const Request = struct {
524524 req.* = undefined;
525525 }
526526
527 pub const Reader = std.io.Reader(*Request, ReadError, read);
528
529 pub fn reader(req: *Request) Reader {
530 return .{ .context = req };
531 }
532
527533 pub fn readAll(req: *Request, buffer: []u8) !usize {
528534 return readAtLeast(req, buffer, buffer.len);
529535 }
530536
531 pub fn read(req: *Request, buffer: []u8) !usize {
537 pub const ReadError = net.Stream.ReadError || error{
538 // From HTTP protocol
539 HttpHeadersInvalid,
540 HttpHeadersExceededSizeLimit,
541 HttpRedirectMissingLocation,
542 HttpTransferEncodingUnsupported,
543 HttpContentLengthUnknown,
544 TooManyHttpRedirects,
545 ShortHttpStatusLine,
546 BadHttpVersion,
547 HttpHeaderContinuationsUnsupported,
548 UnsupportedUrlScheme,
549 UriMissingHost,
550 UnknownHostName,
551
552 // Network problems
553 NetworkUnreachable,
554 HostLacksNetworkAddresses,
555 TemporaryNameServerFailure,
556 NameServerFailure,
557 ProtocolFamilyNotAvailable,
558 ProtocolNotSupported,
559
560 // System resource problems
561 ProcessFdQuotaExceeded,
562 SystemFdQuotaExceeded,
563 OutOfMemory,
564
565 // TLS problems
566 InsufficientEntropy,
567 TlsConnectionTruncated,
568 TlsRecordOverflow,
569 TlsDecodeError,
570 TlsAlert,
571 TlsBadRecordMac,
572 TlsBadLength,
573 TlsIllegalParameter,
574 TlsUnexpectedMessage,
575 TlsDecryptFailure,
576 CertificateFieldHasInvalidLength,
577 CertificateHostMismatch,
578 CertificatePublicKeyInvalid,
579 CertificateExpired,
580 CertificateFieldHasWrongDataType,
581 CertificateIssuerMismatch,
582 CertificateNotYetValid,
583 CertificateSignatureAlgorithmMismatch,
584 CertificateSignatureAlgorithmUnsupported,
585 CertificateSignatureInvalid,
586 CertificateSignatureInvalidLength,
587 CertificateSignatureNamedCurveUnsupported,
588 CertificateSignatureUnsupportedBitCount,
589 TlsCertificateNotVerified,
590 TlsBadSignatureScheme,
591 TlsBadRsaSignatureBitCount,
592 TlsDecryptError,
593 UnsupportedCertificateVersion,
594 CertificateTimeInvalid,
595 CertificateHasUnrecognizedObjectId,
596 CertificateHasInvalidBitString,
597
598 // TODO: convert to higher level errors
599 InvalidFormat,
600 InvalidPort,
601 UnexpectedCharacter,
602 Overflow,
603 InvalidCharacter,
604 AddressFamilyNotSupported,
605 AddressInUse,
606 AddressNotAvailable,
607 ConnectionPending,
608 ConnectionRefused,
609 FileNotFound,
610 PermissionDenied,
611 ServiceUnavailable,
612 SocketTypeNotSupported,
613 FileTooBig,
614 LockViolation,
615 NoSpaceLeft,
616 NotOpenForWriting,
617 InvalidEncoding,
618 IdentityElement,
619 NonCanonical,
620 SignatureVerificationFailed,
621 MessageTooLong,
622 NegativeIntoUnsigned,
623 TargetTooSmall,
624 BufferTooSmall,
625 InvalidSignature,
626 NotSquare,
627 DiskQuota,
628 InvalidEnd,
629 Incomplete,
630 InvalidIpv4Mapping,
631 InvalidIPAddressFormat,
632 BadPathName,
633 DeviceBusy,
634 FileBusy,
635 FileLocksNotSupported,
636 InvalidHandle,
637 InvalidUtf8,
638 NameTooLong,
639 NoDevice,
640 PathAlreadyExists,
641 PipeBusy,
642 SharingViolation,
643 SymLinkLoop,
644 FileSystem,
645 InterfaceNotFound,
646 AlreadyBound,
647 FileDescriptorNotASocket,
648 NetworkSubsystemFailed,
649 NotDir,
650 ReadOnlyFileSystem,
651 };
652
653 pub fn read(req: *Request, buffer: []u8) ReadError!usize {
532654 return readAtLeast(req, buffer, 1);
533655 }
534656
......@@ -671,7 +793,8 @@ pub const Request = struct {
671793 }
672794 },
673795 .chunk_data => {
674 const sub_amt = @min(req.response.next_chunk_length, in.len);
796 // TODO https://github.com/ziglang/zig/issues/14039
797 const sub_amt = @intCast(usize, @min(req.response.next_chunk_length, in.len));
675798 req.response.next_chunk_length -= sub_amt;
676799 if (req.response.next_chunk_length > 0) {
677800 if (in.ptr == buffer.ptr) {
......@@ -709,11 +832,15 @@ pub const Request = struct {
709832 }
710833};
711834
712pub fn deinit(client: *Client, gpa: Allocator) void {
713 client.ca_bundle.deinit(gpa);
835pub fn deinit(client: *Client) void {
836 client.ca_bundle.deinit(client.allocator);
714837 client.* = undefined;
715838}
716839
840pub fn rescanRootCertificates(client: *Client) !void {
841 return client.ca_bundle.rescan(client.allocator);
842}
843
717844pub fn connect(client: *Client, host: []const u8, port: u16, protocol: Connection.Protocol) !Connection {
718845 var conn: Connection = .{
719846 .stream = try net.tcpConnectToHost(client.allocator, host, port),
lib/std/io.zig+1
......@@ -114,6 +114,7 @@ pub const bufferedWriter = @import("io/buffered_writer.zig").bufferedWriter;
114114
115115pub const BufferedReader = @import("io/buffered_reader.zig").BufferedReader;
116116pub const bufferedReader = @import("io/buffered_reader.zig").bufferedReader;
117pub const bufferedReaderSize = @import("io/buffered_reader.zig").bufferedReaderSize;
117118
118119pub const PeekStream = @import("io/peek_stream.zig").PeekStream;
119120pub const peekStream = @import("io/peek_stream.zig").peekStream;
lib/std/io/buffered_reader.zig+6-2
......@@ -45,8 +45,12 @@ pub fn BufferedReader(comptime buffer_size: usize, comptime ReaderType: type) ty
4545 };
4646}
4747
48pub fn bufferedReader(underlying_stream: anytype) BufferedReader(4096, @TypeOf(underlying_stream)) {
49 return .{ .unbuffered_reader = underlying_stream };
48pub fn bufferedReader(reader: anytype) BufferedReader(4096, @TypeOf(reader)) {
49 return .{ .unbuffered_reader = reader };
50}
51
52pub fn bufferedReaderSize(comptime size: usize, reader: anytype) BufferedReader(size, @TypeOf(reader)) {
53 return .{ .unbuffered_reader = reader };
5054}
5155
5256test "io.BufferedReader OneByte" {
lib/std/io/reader.zig+12-2
......@@ -30,10 +30,20 @@ pub fn Reader(
3030 /// means the stream reached the end. Reaching the end of a stream is not an error
3131 /// condition.
3232 pub fn readAll(self: Self, buffer: []u8) Error!usize {
33 return readAtLeast(self, buffer, buffer.len);
34 }
35
36 /// Returns the number of bytes read, calling the underlying read
37 /// function the minimal number of times until the buffer has at least
38 /// `len` bytes filled. If the number read is less than `len` it means
39 /// the stream reached the end. Reaching the end of the stream is not
40 /// an error condition.
41 pub fn readAtLeast(self: Self, buffer: []u8, len: usize) Error!usize {
42 assert(len <= buffer.len);
3343 var index: usize = 0;
34 while (index != buffer.len) {
44 while (index < len) {
3545 const amt = try self.read(buffer[index..]);
36 if (amt == 0) return index;
46 if (amt == 0) break;
3747 index += amt;
3848 }
3949 return index;
lib/std/os.zig+3
......@@ -2414,6 +2414,9 @@ pub fn unlinkatW(dirfd: fd_t, sub_path_w: []const u16, flags: u32) UnlinkatError
24142414pub const RenameError = error{
24152415 /// In WASI, this error may occur when the file descriptor does
24162416 /// not hold the required rights to rename a resource by path relative to it.
2417 ///
2418 /// On Windows, this error may be returned instead of PathAlreadyExists when
2419 /// renaming a directory over an existing directory.
24172420 AccessDenied,
24182421 FileBusy,
24192422 DiskQuota,
lib/std/std.zig+2
......@@ -21,6 +21,7 @@ pub const EnumMap = enums.EnumMap;
2121pub const EnumSet = enums.EnumSet;
2222pub const HashMap = hash_map.HashMap;
2323pub const HashMapUnmanaged = hash_map.HashMapUnmanaged;
24pub const Ini = @import("Ini.zig");
2425pub const MultiArrayList = @import("multi_array_list.zig").MultiArrayList;
2526pub const PackedIntArray = @import("packed_int_array.zig").PackedIntArray;
2627pub const PackedIntArrayEndian = @import("packed_int_array.zig").PackedIntArrayEndian;
......@@ -85,6 +86,7 @@ pub const rand = @import("rand.zig");
8586pub const sort = @import("sort.zig");
8687pub const simd = @import("simd.zig");
8788pub const ascii = @import("ascii.zig");
89pub const tar = @import("tar.zig");
8890pub const testing = @import("testing.zig");
8991pub const time = @import("time.zig");
9092pub const tz = @import("tz.zig");
lib/std/tar.zig created+172
......@@ -0,0 +1,172 @@
1pub const Options = struct {
2 /// Number of directory levels to skip when extracting files.
3 strip_components: u32 = 0,
4};
5
6pub const Header = struct {
7 bytes: *const [512]u8,
8
9 pub const FileType = enum(u8) {
10 normal = '0',
11 hard_link = '1',
12 symbolic_link = '2',
13 character_special = '3',
14 block_special = '4',
15 directory = '5',
16 fifo = '6',
17 contiguous = '7',
18 global_extended_header = 'g',
19 extended_header = 'x',
20 _,
21 };
22
23 pub fn fileSize(header: Header) !u64 {
24 const raw = header.bytes[124..][0..12];
25 const ltrimmed = std.mem.trimLeft(u8, raw, "0");
26 const rtrimmed = std.mem.trimRight(u8, ltrimmed, "\x00");
27 if (rtrimmed.len == 0) return 0;
28 return std.fmt.parseInt(u64, rtrimmed, 8);
29 }
30
31 pub fn is_ustar(header: Header) bool {
32 return std.mem.eql(u8, header.bytes[257..][0..6], "ustar\x00");
33 }
34
35 /// Includes prefix concatenated, if any.
36 /// Return value may point into Header buffer, or might point into the
37 /// argument buffer.
38 /// TODO: check against "../" and other nefarious things
39 pub fn fullFileName(header: Header, buffer: *[255]u8) ![]const u8 {
40 const n = name(header);
41 if (!is_ustar(header))
42 return n;
43 const p = prefix(header);
44 if (p.len == 0)
45 return n;
46 std.mem.copy(u8, buffer[0..p.len], p);
47 buffer[p.len] = '/';
48 std.mem.copy(u8, buffer[p.len + 1 ..], n);
49 return buffer[0 .. p.len + 1 + n.len];
50 }
51
52 pub fn name(header: Header) []const u8 {
53 return str(header, 0, 0 + 100);
54 }
55
56 pub fn prefix(header: Header) []const u8 {
57 return str(header, 345, 345 + 155);
58 }
59
60 pub fn fileType(header: Header) FileType {
61 const result = @intToEnum(FileType, header.bytes[156]);
62 return if (result == @intToEnum(FileType, 0)) .normal else result;
63 }
64
65 fn str(header: Header, start: usize, end: usize) []const u8 {
66 var i: usize = start;
67 while (i < end) : (i += 1) {
68 if (header.bytes[i] == 0) break;
69 }
70 return header.bytes[start..i];
71 }
72};
73
74pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !void {
75 var file_name_buffer: [255]u8 = undefined;
76 var buffer: [512 * 8]u8 = undefined;
77 var start: usize = 0;
78 var end: usize = 0;
79 header: while (true) {
80 if (buffer.len - start < 1024) {
81 std.mem.copy(u8, &buffer, buffer[start..end]);
82 end -= start;
83 start = 0;
84 }
85 const ask_header = @min(buffer.len - end, 1024 -| (end - start));
86 end += try reader.readAtLeast(buffer[end..], ask_header);
87 switch (end - start) {
88 0 => return,
89 1...511 => return error.UnexpectedEndOfStream,
90 else => {},
91 }
92 const header: Header = .{ .bytes = buffer[start..][0..512] };
93 start += 512;
94 const file_size = try header.fileSize();
95 const rounded_file_size = std.mem.alignForwardGeneric(u64, file_size, 512);
96 const pad_len = @intCast(usize, rounded_file_size - file_size);
97 const unstripped_file_name = try header.fullFileName(&file_name_buffer);
98 switch (header.fileType()) {
99 .directory => {
100 const file_name = try stripComponents(unstripped_file_name, options.strip_components);
101 if (file_name.len != 0) {
102 try dir.makeDir(file_name);
103 }
104 },
105 .normal => {
106 if (file_size == 0 and unstripped_file_name.len == 0) return;
107 const file_name = try stripComponents(unstripped_file_name, options.strip_components);
108
109 var file = try dir.createFile(file_name, .{});
110 defer file.close();
111
112 var file_off: usize = 0;
113 while (true) {
114 if (buffer.len - start < 1024) {
115 std.mem.copy(u8, &buffer, buffer[start..end]);
116 end -= start;
117 start = 0;
118 }
119 // Ask for the rounded up file size + 512 for the next header.
120 // TODO: https://github.com/ziglang/zig/issues/14039
121 const ask = @intCast(usize, @min(
122 buffer.len - end,
123 rounded_file_size + 512 - file_off -| (end - start),
124 ));
125 end += try reader.readAtLeast(buffer[end..], ask);
126 if (end - start < ask) return error.UnexpectedEndOfStream;
127 // TODO: https://github.com/ziglang/zig/issues/14039
128 const slice = buffer[start..@intCast(usize, @min(file_size - file_off + start, end))];
129 try file.writeAll(slice);
130 file_off += slice.len;
131 start += slice.len;
132 if (file_off >= file_size) {
133 start += pad_len;
134 // Guaranteed since we use a buffer divisible by 512.
135 assert(start <= end);
136 continue :header;
137 }
138 }
139 },
140 .global_extended_header, .extended_header => {
141 if (start + rounded_file_size > end) return error.TarHeadersTooBig;
142 start = @intCast(usize, start + rounded_file_size);
143 },
144 .hard_link => return error.TarUnsupportedFileType,
145 .symbolic_link => return error.TarUnsupportedFileType,
146 else => return error.TarUnsupportedFileType,
147 }
148 }
149}
150
151fn stripComponents(path: []const u8, count: u32) ![]const u8 {
152 var i: usize = 0;
153 var c = count;
154 while (c > 0) : (c -= 1) {
155 if (std.mem.indexOfScalarPos(u8, path, i, '/')) |pos| {
156 i = pos + 1;
157 } else {
158 return error.TarComponentsOutsideStrippedPrefix;
159 }
160 }
161 return path[i..];
162}
163
164test stripComponents {
165 const expectEqualStrings = std.testing.expectEqualStrings;
166 try expectEqualStrings("a/b/c", try stripComponents("a/b/c", 0));
167 try expectEqualStrings("b/c", try stripComponents("a/b/c", 1));
168 try expectEqualStrings("c", try stripComponents("a/b/c", 2));
169}
170
171const std = @import("std.zig");
172const assert = std.debug.assert;
src/Package.zig+457
......@@ -5,9 +5,15 @@ const fs = std.fs;
55const mem = std.mem;
66const Allocator = mem.Allocator;
77const assert = std.debug.assert;
8const Hash = std.crypto.hash.sha2.Sha256;
9const log = std.log.scoped(.package);
810
911const Compilation = @import("Compilation.zig");
1012const Module = @import("Module.zig");
13const ThreadPool = @import("ThreadPool.zig");
14const WaitGroup = @import("WaitGroup.zig");
15const Cache = @import("Cache.zig");
16const build_options = @import("build_options");
1117
1218pub const Table = std.StringHashMapUnmanaged(*Package);
1319
......@@ -124,3 +130,454 @@ pub fn addAndAdopt(parent: *Package, gpa: Allocator, name: []const u8, child: *P
124130 child.parent = parent;
125131 return parent.add(gpa, name, child);
126132}
133
134pub const build_zig_basename = "build.zig";
135pub const ini_basename = build_zig_basename ++ ".ini";
136
137pub fn fetchAndAddDependencies(
138 pkg: *Package,
139 thread_pool: *ThreadPool,
140 http_client: *std.http.Client,
141 directory: Compilation.Directory,
142 global_cache_directory: Compilation.Directory,
143 local_cache_directory: Compilation.Directory,
144 dependencies_source: *std.ArrayList(u8),
145 build_roots_source: *std.ArrayList(u8),
146 name_prefix: []const u8,
147) !void {
148 const max_bytes = 10 * 1024 * 1024;
149 const gpa = thread_pool.allocator;
150 const build_zig_ini = directory.handle.readFileAlloc(gpa, ini_basename, max_bytes) catch |err| switch (err) {
151 error.FileNotFound => {
152 // Handle the same as no dependencies.
153 return;
154 },
155 else => |e| return e,
156 };
157 defer gpa.free(build_zig_ini);
158
159 const ini: std.Ini = .{ .bytes = build_zig_ini };
160 var any_error = false;
161 var it = ini.iterateSection("\n[dependency]\n");
162 while (it.next()) |dep| {
163 var line_it = mem.split(u8, dep, "\n");
164 var opt_name: ?[]const u8 = null;
165 var opt_url: ?[]const u8 = null;
166 var expected_hash: ?[]const u8 = null;
167 while (line_it.next()) |kv| {
168 const eq_pos = mem.indexOfScalar(u8, kv, '=') orelse continue;
169 const key = kv[0..eq_pos];
170 const value = kv[eq_pos + 1 ..];
171 if (mem.eql(u8, key, "name")) {
172 opt_name = value;
173 } else if (mem.eql(u8, key, "url")) {
174 opt_url = value;
175 } else if (mem.eql(u8, key, "hash")) {
176 expected_hash = value;
177 } else {
178 const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(key.ptr) - @ptrToInt(ini.bytes.ptr));
179 std.log.warn("{s}/{s}:{d}:{d} unrecognized key: '{s}'", .{
180 directory.path orelse ".",
181 "build.zig.ini",
182 loc.line,
183 loc.column,
184 key,
185 });
186 }
187 }
188
189 const name = opt_name orelse {
190 const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr));
191 std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{
192 directory.path orelse ".",
193 "build.zig.ini",
194 loc.line,
195 loc.column,
196 });
197 any_error = true;
198 continue;
199 };
200
201 const url = opt_url orelse {
202 const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(dep.ptr) - @ptrToInt(ini.bytes.ptr));
203 std.log.err("{s}/{s}:{d}:{d} missing key: 'name'", .{
204 directory.path orelse ".",
205 "build.zig.ini",
206 loc.line,
207 loc.column,
208 });
209 any_error = true;
210 continue;
211 };
212
213 const sub_prefix = try std.fmt.allocPrint(gpa, "{s}{s}.", .{ name_prefix, name });
214 defer gpa.free(sub_prefix);
215 const fqn = sub_prefix[0 .. sub_prefix.len - 1];
216
217 const sub_pkg = try fetchAndUnpack(
218 thread_pool,
219 http_client,
220 global_cache_directory,
221 url,
222 expected_hash,
223 ini,
224 directory,
225 build_roots_source,
226 fqn,
227 );
228
229 try pkg.fetchAndAddDependencies(
230 thread_pool,
231 http_client,
232 sub_pkg.root_src_directory,
233 global_cache_directory,
234 local_cache_directory,
235 dependencies_source,
236 build_roots_source,
237 sub_prefix,
238 );
239
240 try addAndAdopt(pkg, gpa, fqn, sub_pkg);
241
242 try dependencies_source.writer().print(" pub const {s} = @import(\"{}\");\n", .{
243 std.zig.fmtId(fqn), std.zig.fmtEscapes(fqn),
244 });
245 }
246
247 if (any_error) return error.InvalidBuildZigIniFile;
248}
249
250pub fn createFilePkg(
251 gpa: Allocator,
252 cache_directory: Compilation.Directory,
253 basename: []const u8,
254 contents: []const u8,
255) !*Package {
256 const rand_int = std.crypto.random.int(u64);
257 const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ hex64(rand_int);
258 {
259 var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});
260 defer tmp_dir.close();
261 try tmp_dir.writeFile(basename, contents);
262 }
263
264 var hh: Cache.HashHelper = .{};
265 hh.addBytes(build_options.version);
266 hh.addBytes(contents);
267 const hex_digest = hh.final();
268
269 const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest;
270 try renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path);
271
272 return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
273}
274
275fn fetchAndUnpack(
276 thread_pool: *ThreadPool,
277 http_client: *std.http.Client,
278 global_cache_directory: Compilation.Directory,
279 url: []const u8,
280 expected_hash: ?[]const u8,
281 ini: std.Ini,
282 comp_directory: Compilation.Directory,
283 build_roots_source: *std.ArrayList(u8),
284 fqn: []const u8,
285) !*Package {
286 const gpa = http_client.allocator;
287 const s = fs.path.sep_str;
288
289 // Check if the expected_hash is already present in the global package
290 // cache, and thereby avoid both fetching and unpacking.
291 if (expected_hash) |h| cached: {
292 if (h.len != 2 * Hash.digest_length) {
293 return reportError(
294 ini,
295 comp_directory,
296 h.ptr,
297 "wrong hash size. expected: {d}, found: {d}",
298 .{ Hash.digest_length, h.len },
299 );
300 }
301 const hex_digest = h[0 .. 2 * Hash.digest_length];
302 const pkg_dir_sub_path = "p" ++ s ++ hex_digest;
303 var pkg_dir = global_cache_directory.handle.openDir(pkg_dir_sub_path, .{}) catch |err| switch (err) {
304 error.FileNotFound => break :cached,
305 else => |e| return e,
306 };
307 errdefer pkg_dir.close();
308
309 const ptr = try gpa.create(Package);
310 errdefer gpa.destroy(ptr);
311
312 const owned_src_path = try gpa.dupe(u8, build_zig_basename);
313 errdefer gpa.free(owned_src_path);
314
315 const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
316 errdefer gpa.free(build_root);
317
318 try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{
319 std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
320 });
321
322 ptr.* = .{
323 .root_src_directory = .{
324 .path = build_root,
325 .handle = pkg_dir,
326 },
327 .root_src_directory_owned = true,
328 .root_src_path = owned_src_path,
329 };
330
331 return ptr;
332 }
333
334 const uri = try std.Uri.parse(url);
335
336 const rand_int = std.crypto.random.int(u64);
337 const tmp_dir_sub_path = "tmp" ++ s ++ hex64(rand_int);
338
339 const actual_hash = a: {
340 var tmp_directory: Compilation.Directory = d: {
341 const path = try global_cache_directory.join(gpa, &.{tmp_dir_sub_path});
342 errdefer gpa.free(path);
343
344 const iterable_dir = try global_cache_directory.handle.makeOpenPathIterable(tmp_dir_sub_path, .{});
345 errdefer iterable_dir.close();
346
347 break :d .{
348 .path = path,
349 .handle = iterable_dir.dir,
350 };
351 };
352 defer tmp_directory.closeAndFree(gpa);
353
354 var req = try http_client.request(uri, .{}, .{});
355 defer req.deinit();
356
357 if (mem.endsWith(u8, uri.path, ".tar.gz")) {
358 // I observed the gzip stream to read 1 byte at a time, so I am using a
359 // buffered reader on the front of it.
360 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader());
361
362 var gzip_stream = try std.compress.gzip.gzipStream(gpa, br.reader());
363 defer gzip_stream.deinit();
364
365 try std.tar.pipeToFileSystem(tmp_directory.handle, gzip_stream.reader(), .{
366 .strip_components = 1,
367 });
368 } else {
369 return reportError(
370 ini,
371 comp_directory,
372 uri.path.ptr,
373 "unknown file extension for path '{s}'",
374 .{uri.path},
375 );
376 }
377
378 // TODO: delete files not included in the package prior to computing the package hash.
379 // for example, if the ini file has directives to include/not include certain files,
380 // apply those rules directly to the filesystem right here. This ensures that files
381 // not protected by the hash are not present on the file system.
382
383 break :a try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle });
384 };
385
386 const pkg_dir_sub_path = "p" ++ s ++ hexDigest(actual_hash);
387 try renameTmpIntoCache(global_cache_directory.handle, tmp_dir_sub_path, pkg_dir_sub_path);
388
389 if (expected_hash) |h| {
390 const actual_hex = hexDigest(actual_hash);
391 if (!mem.eql(u8, h, &actual_hex)) {
392 return reportError(
393 ini,
394 comp_directory,
395 h.ptr,
396 "hash mismatch: expected: {s}, found: {s}",
397 .{ h, actual_hex },
398 );
399 }
400 } else {
401 return reportError(
402 ini,
403 comp_directory,
404 url.ptr,
405 "url field is missing corresponding hash field: hash={s}",
406 .{std.fmt.fmtSliceHexLower(&actual_hash)},
407 );
408 }
409
410 const build_root = try global_cache_directory.join(gpa, &.{pkg_dir_sub_path});
411 defer gpa.free(build_root);
412
413 try build_roots_source.writer().print(" pub const {s} = \"{}\";\n", .{
414 std.zig.fmtId(fqn), std.zig.fmtEscapes(build_root),
415 });
416
417 return createWithDir(gpa, global_cache_directory, pkg_dir_sub_path, build_zig_basename);
418}
419
420fn reportError(
421 ini: std.Ini,
422 comp_directory: Compilation.Directory,
423 src_ptr: [*]const u8,
424 comptime fmt_string: []const u8,
425 fmt_args: anytype,
426) error{PackageFetchFailed} {
427 const loc = std.zig.findLineColumn(ini.bytes, @ptrToInt(src_ptr) - @ptrToInt(ini.bytes.ptr));
428 if (comp_directory.path) |p| {
429 std.debug.print("{s}{c}{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{
430 p, fs.path.sep, ini_basename, loc.line + 1, loc.column + 1,
431 } ++ fmt_args);
432 } else {
433 std.debug.print("{s}:{d}:{d}: error: " ++ fmt_string ++ "\n", .{
434 ini_basename, loc.line + 1, loc.column + 1,
435 } ++ fmt_args);
436 }
437 return error.PackageFetchFailed;
438}
439
440const HashedFile = struct {
441 path: []const u8,
442 hash: [Hash.digest_length]u8,
443 failure: Error!void,
444
445 const Error = fs.File.OpenError || fs.File.ReadError;
446
447 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
448 _ = context;
449 return mem.lessThan(u8, lhs.path, rhs.path);
450 }
451};
452
453fn computePackageHash(
454 thread_pool: *ThreadPool,
455 pkg_dir: fs.IterableDir,
456) ![Hash.digest_length]u8 {
457 const gpa = thread_pool.allocator;
458
459 // We'll use an arena allocator for the path name strings since they all
460 // need to be in memory for sorting.
461 var arena_instance = std.heap.ArenaAllocator.init(gpa);
462 defer arena_instance.deinit();
463 const arena = arena_instance.allocator();
464
465 // Collect all files, recursively, then sort.
466 var all_files = std.ArrayList(*HashedFile).init(gpa);
467 defer all_files.deinit();
468
469 var walker = try pkg_dir.walk(gpa);
470 defer walker.deinit();
471
472 {
473 // The final hash will be a hash of each file hashed independently. This
474 // allows hashing in parallel.
475 var wait_group: WaitGroup = .{};
476 defer wait_group.wait();
477
478 while (try walker.next()) |entry| {
479 switch (entry.kind) {
480 .Directory => continue,
481 .File => {},
482 else => return error.IllegalFileTypeInPackage,
483 }
484 const hashed_file = try arena.create(HashedFile);
485 hashed_file.* = .{
486 .path = try arena.dupe(u8, entry.path),
487 .hash = undefined, // to be populated by the worker
488 .failure = undefined, // to be populated by the worker
489 };
490 wait_group.start();
491 try thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
492
493 try all_files.append(hashed_file);
494 }
495 }
496
497 std.sort.sort(*HashedFile, all_files.items, {}, HashedFile.lessThan);
498
499 var hasher = Hash.init(.{});
500 var any_failures = false;
501 for (all_files.items) |hashed_file| {
502 hashed_file.failure catch |err| {
503 any_failures = true;
504 std.log.err("unable to hash '{s}': {s}", .{ hashed_file.path, @errorName(err) });
505 };
506 hasher.update(&hashed_file.hash);
507 }
508 if (any_failures) return error.PackageHashUnavailable;
509 return hasher.finalResult();
510}
511
512fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
513 defer wg.finish();
514 hashed_file.failure = hashFileFallible(dir, hashed_file);
515}
516
517fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
518 var buf: [8000]u8 = undefined;
519 var file = try dir.openFile(hashed_file.path, .{});
520 var hasher = Hash.init(.{});
521 while (true) {
522 const bytes_read = try file.read(&buf);
523 if (bytes_read == 0) break;
524 hasher.update(buf[0..bytes_read]);
525 }
526 hasher.final(&hashed_file.hash);
527}
528
529const hex_charset = "0123456789abcdef";
530
531fn hex64(x: u64) [16]u8 {
532 var result: [16]u8 = undefined;
533 var i: usize = 0;
534 while (i < 8) : (i += 1) {
535 const byte = @truncate(u8, x >> @intCast(u6, 8 * i));
536 result[i * 2 + 0] = hex_charset[byte >> 4];
537 result[i * 2 + 1] = hex_charset[byte & 15];
538 }
539 return result;
540}
541
542test hex64 {
543 const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]";
544 try std.testing.expectEqualStrings("[00efcdab78563412]", s);
545}
546
547fn hexDigest(digest: [Hash.digest_length]u8) [Hash.digest_length * 2]u8 {
548 var result: [Hash.digest_length * 2]u8 = undefined;
549 for (digest) |byte, i| {
550 result[i * 2 + 0] = hex_charset[byte >> 4];
551 result[i * 2 + 1] = hex_charset[byte & 15];
552 }
553 return result;
554}
555
556fn renameTmpIntoCache(
557 cache_dir: fs.Dir,
558 tmp_dir_sub_path: []const u8,
559 dest_dir_sub_path: []const u8,
560) !void {
561 assert(dest_dir_sub_path[1] == fs.path.sep);
562 var handled_missing_dir = false;
563 while (true) {
564 cache_dir.rename(tmp_dir_sub_path, dest_dir_sub_path) catch |err| switch (err) {
565 error.FileNotFound => {
566 if (handled_missing_dir) return err;
567 cache_dir.makeDir(dest_dir_sub_path[0..1]) catch |mkd_err| switch (mkd_err) {
568 error.PathAlreadyExists => handled_missing_dir = true,
569 else => |e| return e,
570 };
571 continue;
572 },
573 error.PathAlreadyExists, error.AccessDenied => {
574 // Package has been already downloaded and may already be in use on the system.
575 cache_dir.deleteTree(tmp_dir_sub_path) catch |del_err| {
576 std.log.warn("unable to delete temp directory: {s}", .{@errorName(del_err)});
577 };
578 },
579 else => |e| return e,
580 };
581 break;
582 }
583}
src/main.zig+60-11
......@@ -3983,11 +3983,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
39833983 };
39843984 defer zig_lib_directory.handle.close();
39853985
3986 var main_pkg: Package = .{
3987 .root_src_directory = zig_lib_directory,
3988 .root_src_path = "build_runner.zig",
3989 };
3990
39913986 var cleanup_build_dir: ?fs.Dir = null;
39923987 defer if (cleanup_build_dir) |*dir| dir.close();
39933988
......@@ -4031,12 +4026,6 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
40314026 };
40324027 child_argv.items[argv_index_build_file] = build_directory.path orelse cwd_path;
40334028
4034 var build_pkg: Package = .{
4035 .root_src_directory = build_directory,
4036 .root_src_path = build_zig_basename,
4037 };
4038 try main_pkg.addAndAdopt(arena, "@build", &build_pkg);
4039
40404029 var global_cache_directory: Compilation.Directory = l: {
40414030 const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
40424031 break :l .{
......@@ -4082,6 +4071,66 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
40824071 var thread_pool: ThreadPool = undefined;
40834072 try thread_pool.init(gpa);
40844073 defer thread_pool.deinit();
4074
4075 var main_pkg: Package = .{
4076 .root_src_directory = zig_lib_directory,
4077 .root_src_path = "build_runner.zig",
4078 };
4079
4080 if (!build_options.omit_pkg_fetching_code) {
4081 var http_client: std.http.Client = .{ .allocator = gpa };
4082 defer http_client.deinit();
4083 try http_client.rescanRootCertificates();
4084
4085 // Here we provide an import to the build runner that allows using reflection to find
4086 // all of the dependencies. Without this, there would be no way to use `@import` to
4087 // access dependencies by name, since `@import` requires string literals.
4088 var dependencies_source = std.ArrayList(u8).init(gpa);
4089 defer dependencies_source.deinit();
4090 try dependencies_source.appendSlice("pub const imports = struct {\n");
4091
4092 // This will go into the same package. It contains the file system paths
4093 // to all the build.zig files.
4094 var build_roots_source = std.ArrayList(u8).init(gpa);
4095 defer build_roots_source.deinit();
4096
4097 // Here we borrow main package's table and will replace it with a fresh
4098 // one after this process completes.
4099 main_pkg.fetchAndAddDependencies(
4100 &thread_pool,
4101 &http_client,
4102 build_directory,
4103 global_cache_directory,
4104 local_cache_directory,
4105 &dependencies_source,
4106 &build_roots_source,
4107 "",
4108 ) catch |err| switch (err) {
4109 error.PackageFetchFailed => process.exit(1),
4110 else => |e| return e,
4111 };
4112
4113 try dependencies_source.appendSlice("};\npub const build_root = struct {\n");
4114 try dependencies_source.appendSlice(build_roots_source.items);
4115 try dependencies_source.appendSlice("};\n");
4116
4117 const deps_pkg = try Package.createFilePkg(
4118 gpa,
4119 local_cache_directory,
4120 "dependencies.zig",
4121 dependencies_source.items,
4122 );
4123
4124 mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table);
4125 try main_pkg.addAndAdopt(gpa, "@dependencies", deps_pkg);
4126 }
4127
4128 var build_pkg: Package = .{
4129 .root_src_directory = build_directory,
4130 .root_src_path = build_zig_basename,
4131 };
4132 try main_pkg.addAndAdopt(gpa, "@build", &build_pkg);
4133
40854134 const comp = Compilation.create(gpa, .{
40864135 .zig_lib_directory = zig_lib_directory,
40874136 .local_cache_directory = local_cache_directory,
stage1/config.zig.in+1
......@@ -12,3 +12,4 @@ pub const have_stage1 = false;
1212pub const skip_non_native = false;
1313pub const only_c = false;
1414pub const force_gpa = false;
15pub const omit_pkg_fetching_code = true;