| ... | ... | @@ -1,28 +1,34 @@ |
| 1 | 1 | //! Represents one independent job whose responsibility is to: |
| 2 | 2 | //! |
| 3 | | //! 1. Check the global zig package cache to see if the hash already exists. |
| 3 | //! 1. Check the local zig package directory to see if the hash already exists. |
| 4 | 4 | //! If so, load, parse, and validate the build.zig.zon file therein, and |
| 5 | | //! goto step 8. Likewise if the location is a relative path, treat this |
| 5 | //! goto step 9. Likewise if the location is a relative path, treat this |
| 6 | 6 | //! the same as a cache hit. Otherwise, proceed. |
| 7 | | //! 2. Fetch and unpack a URL into a temporary directory. |
| 8 | | //! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed |
| 7 | //! 2. Check the global package cache for a compressed tarball matching the |
| 8 | //! hash. If it is found, unpack the contents into a temporary directory inside |
| 9 | //! project local zig cache. Rename this directory into the local zig package |
| 10 | //! directory and goto step 9, skipping step 10. |
| 11 | //! 3. Fetch and unpack a URL into a temporary directory. |
| 12 | //! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed |
| 9 | 13 | //! for the file to be missing, in which case this fetched package is considered |
| 10 | 14 | //! to be a "naked" package. |
| 11 | | //! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by |
| 15 | //! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by |
| 12 | 16 | //! deleting excluded files. If any files had errors for files that were |
| 13 | 17 | //! ultimately excluded, those errors should be ignored, such as failure to |
| 14 | 18 | //! create symlinks that weren't supposed to be included anyway. |
| 15 | | //! 5. Compute the package hash based on the remaining files in the temporary |
| 19 | //! 6. Compute the package hash based on the remaining files in the temporary |
| 16 | 20 | //! directory. |
| 17 | | //! 6. Rename the temporary directory into the global zig package cache |
| 18 | | //! directory. If the hash already exists, delete the temporary directory and |
| 19 | | //! leave the zig package cache directory untouched as it may be in use by the |
| 20 | | //! system. This is done even if the hash is invalid, in case the package with |
| 21 | | //! the different hash is used in the future. |
| 22 | | //! 7. Validate the computed hash against the expected hash. If invalid, |
| 21 | //! 7. Rename the temporary directory into the local zig package directory. If |
| 22 | //! the hash already exists, delete the temporary directory and leave the zig |
| 23 | //! package directory untouched as it may be in use. This is done even if |
| 24 | //! the hash is invalid, in case the package with the different hash is used |
| 25 | //! in the future. |
| 26 | //! 8. Validate the computed hash against the expected hash. If invalid, |
| 23 | 27 | //! this job is done. |
| 24 | | //! 8. Spawn a new fetch job for each dependency in the manifest file. Use |
| 28 | //! 9. Spawn a new fetch job for each dependency in the manifest file. Use |
| 25 | 29 | //! a mutex and a hash map so that redundant jobs do not get queued up. |
| 30 | //! 10.Compress the package directory and store it into the global package |
| 31 | //! cache. |
| 26 | 32 | //! |
| 27 | 33 | //! All of this must be done with only referring to the state inside this struct |
| 28 | 34 | //! because this work will be done in a dedicated thread. |
| ... | ... | @@ -34,6 +40,7 @@ const native_os = builtin.os.tag; |
| 34 | 40 | const std = @import("std"); |
| 35 | 41 | const Io = std.Io; |
| 36 | 42 | const fs = std.fs; |
| 43 | const log = std.log.scoped(.fetch); |
| 37 | 44 | const assert = std.debug.assert; |
| 38 | 45 | const ascii = std.ascii; |
| 39 | 46 | const Allocator = std.mem.Allocator; |
| ... | ... | @@ -60,16 +67,13 @@ omit_missing_hash_error: bool, |
| 60 | 67 | /// which specifies inclusion rules. This is intended to be true for the first |
| 61 | 68 | /// fetch task and false for the recursive dependencies. |
| 62 | 69 | allow_missing_paths_field: bool, |
| 63 | | allow_missing_fingerprint: bool, |
| 64 | | allow_name_string: bool, |
| 65 | 70 | /// If true and URL points to a Git repository, will use the latest commit. |
| 66 | 71 | use_latest_commit: bool, |
| 67 | 72 | |
| 68 | 73 | // Above this are fields provided as inputs to `run`. |
| 69 | 74 | // Below this are fields populated by `run`. |
| 70 | 75 | |
| 71 | | /// This will either be relative to `global_cache`, or to the build root of |
| 72 | | /// the root package. |
| 76 | /// Relative to the build root of the root package. |
| 73 | 77 | package_root: Cache.Path, |
| 74 | 78 | error_bundle: ErrorBundle.Wip, |
| 75 | 79 | manifest: ?Manifest, |
| ... | ... | @@ -111,10 +115,15 @@ pub const JobQueue = struct { |
| 111 | 115 | /// field contains references to all of them. |
| 112 | 116 | /// Protected by `mutex`. |
| 113 | 117 | all_fetches: std.ArrayList(*Fetch) = .empty, |
| 118 | prog_node: std.Progress.Node, |
| 114 | 119 | |
| 115 | 120 | http_client: *std.http.Client, |
| 121 | /// This tracks `Fetch` tasks as well as recompression tasks. |
| 116 | 122 | group: Io.Group = .init, |
| 117 | 123 | global_cache: Cache.Directory, |
| 124 | local_cache: Cache.Path, |
| 125 | /// Path to "zig-pkg" inside the package in which the user ran `zig build`. |
| 126 | root_pkg_path: Cache.Path, |
| 118 | 127 | /// If true then, no fetching occurs, and: |
| 119 | 128 | /// * The `global_cache` directory is assumed to be the direct parent |
| 120 | 129 | /// directory of on-disk packages rather than having the "p/" directory |
| ... | ... | @@ -129,7 +138,6 @@ pub const JobQueue = struct { |
| 129 | 138 | /// two hashes of the same package do not match. |
| 130 | 139 | /// If this is true, `recursive` must be false. |
| 131 | 140 | debug_hash: bool, |
| 132 | | work_around_btrfs_bug: bool, |
| 133 | 141 | mode: Mode, |
| 134 | 142 | /// Set of hashes that will be additionally fetched even if they are marked |
| 135 | 143 | /// as lazy. |
| ... | ... | @@ -294,8 +302,121 @@ pub const JobQueue = struct { |
| 294 | 302 | \\ |
| 295 | 303 | ); |
| 296 | 304 | } |
| 305 | |
| 306 | fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void { |
| 307 | const pkg_hash_slice = package_hash.toSlice(); |
| 308 | |
| 309 | const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice}); |
| 310 | defer prog_node.end(); |
| 311 | |
| 312 | var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined; |
| 313 | const dest_path: Cache.Path = .{ |
| 314 | .root_dir = jq.global_cache, |
| 315 | .sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable, |
| 316 | }; |
| 317 | |
| 318 | const gpa = jq.http_client.allocator; |
| 319 | |
| 320 | var arena_instance = std.heap.ArenaAllocator.init(gpa); |
| 321 | defer arena_instance.deinit(); |
| 322 | const arena = arena_instance.allocator(); |
| 323 | |
| 324 | recompressFallible(jq, arena, dest_path, pkg_hash_slice, prog_node) catch |err| switch (err) { |
| 325 | error.Canceled => |e| return e, |
| 326 | error.ReadFailed => comptime unreachable, |
| 327 | error.WriteFailed => comptime unreachable, |
| 328 | else => |e| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }), |
| 329 | }; |
| 330 | } |
| 331 | |
| 332 | fn recompressFallible( |
| 333 | jq: *JobQueue, |
| 334 | arena: Allocator, |
| 335 | dest_path: Cache.Path, |
| 336 | pkg_hash_slice: []const u8, |
| 337 | prog_node: std.Progress.Node, |
| 338 | ) !void { |
| 339 | const gpa = jq.http_client.allocator; |
| 340 | const io = jq.io; |
| 341 | |
| 342 | // We have to walk the file system up front in order to sort the file |
| 343 | // list for determinism purposes. The hash of the recompressed file is |
| 344 | // not critical because the true hash is based on the content alone. |
| 345 | // However, if we want Zig users to be able to share cached package |
| 346 | // data with each other via peer-to-peer protocols, we benefit greatly |
| 347 | // from the data being identical on everyone's computers. |
| 348 | var scanned_files: std.ArrayList([]const u8) = .empty; |
| 349 | defer scanned_files.deinit(gpa); |
| 350 | |
| 351 | var pkg_dir = try jq.root_pkg_path.openDir(io, pkg_hash_slice, .{ .iterate = true }); |
| 352 | defer pkg_dir.close(io); |
| 353 | |
| 354 | { |
| 355 | var walker = try pkg_dir.walk(gpa); |
| 356 | defer walker.deinit(); |
| 357 | |
| 358 | while (try walker.next(io)) |entry| { |
| 359 | switch (entry.kind) { |
| 360 | .directory => continue, |
| 361 | .file, .sym_link => {}, |
| 362 | else => { |
| 363 | return error.IllegalFileType; |
| 364 | }, |
| 365 | } |
| 366 | const entry_path = try arena.dupe(u8, entry.path); |
| 367 | try scanned_files.append(gpa, entry_path); |
| 368 | } |
| 369 | |
| 370 | std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp); |
| 371 | } |
| 372 | |
| 373 | prog_node.setEstimatedTotalItems(scanned_files.items.len); |
| 374 | |
| 375 | var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{ |
| 376 | .make_path = true, |
| 377 | .replace = true, |
| 378 | }); |
| 379 | defer atomic_file.deinit(io); |
| 380 | |
| 381 | var file_write_buffer: [4096]u8 = undefined; |
| 382 | var file_writer = atomic_file.file.writer(io, &file_write_buffer); |
| 383 | |
| 384 | var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined; |
| 385 | var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) { |
| 386 | error.WriteFailed => return file_writer.err.?, |
| 387 | }; |
| 388 | |
| 389 | var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer }; |
| 390 | archiver.prefix = pkg_hash_slice; |
| 391 | |
| 392 | var file_read_buffer: [4096]u8 = undefined; |
| 393 | |
| 394 | for (scanned_files.items) |entry_path| { |
| 395 | var file = try pkg_dir.openFile(io, entry_path, .{}); |
| 396 | defer file.close(io); |
| 397 | var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer); |
| 398 | archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) { |
| 399 | error.ReadFailed => return file_reader.err.?, |
| 400 | error.WriteFailed => return file_writer.err.?, |
| 401 | else => |e| return e, |
| 402 | }; |
| 403 | prog_node.completeOne(); |
| 404 | } |
| 405 | |
| 406 | // intentionally omitting the pointless trailer |
| 407 | //try archiver.finish(); |
| 408 | compress.writer.flush() catch |err| switch (err) { |
| 409 | error.WriteFailed => return file_writer.err.?, |
| 410 | }; |
| 411 | try file_writer.flush(); |
| 412 | try atomic_file.replace(io); |
| 413 | } |
| 297 | 414 | }; |
| 298 | 415 | |
| 416 | fn stringCmp(_: void, lhs: []const u8, rhs: []const u8) bool { |
| 417 | return std.mem.lessThan(u8, lhs, rhs); |
| 418 | } |
| 419 | |
| 299 | 420 | pub const Location = union(enum) { |
| 300 | 421 | remote: Remote, |
| 301 | 422 | /// A directory found inside the parent package. |
| ... | ... | @@ -326,11 +447,12 @@ pub const RunError = error{ |
| 326 | 447 | }; |
| 327 | 448 | |
| 328 | 449 | pub fn run(f: *Fetch) RunError!void { |
| 329 | | const io = f.job_queue.io; |
| 450 | const job_queue = f.job_queue; |
| 451 | const io = job_queue.io; |
| 330 | 452 | const eb = &f.error_bundle; |
| 331 | 453 | const arena = f.arena.allocator(); |
| 332 | 454 | const gpa = f.arena.child_allocator; |
| 333 | | const cache_root = f.job_queue.global_cache; |
| 455 | const local_cache_root = job_queue.local_cache; |
| 334 | 456 | |
| 335 | 457 | try eb.init(gpa); |
| 336 | 458 | |
| ... | ... | @@ -351,13 +473,13 @@ pub fn run(f: *Fetch) RunError!void { |
| 351 | 473 | ); |
| 352 | 474 | // Packages fetched by URL may not use relative paths to escape outside the |
| 353 | 475 | // fetched package directory from within the package cache. |
| 354 | | if (pkg_root.root_dir.eql(cache_root)) { |
| 476 | if (pkg_root.root_dir.eql(local_cache_root.root_dir)) { |
| 355 | 477 | // `parent_package_root.sub_path` contains a path like this: |
| 356 | 478 | // "p/$hash", or |
| 357 | 479 | // "p/$hash/foo", with possibly more directories after "foo". |
| 358 | 480 | // We want to fail unless the resolved relative path has a |
| 359 | 481 | // prefix of "p/$hash/". |
| 360 | | const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len; |
| 482 | const prefix_len: usize = if (job_queue.read_only) 0 else "p/".len; |
| 361 | 483 | const parent_sub_path = f.parent_package_root.sub_path; |
| 362 | 484 | const end = find_end: { |
| 363 | 485 | if (parent_sub_path.len > prefix_len) { |
| ... | ... | @@ -380,21 +502,21 @@ pub fn run(f: *Fetch) RunError!void { |
| 380 | 502 | f.package_root = pkg_root; |
| 381 | 503 | try loadManifest(f, pkg_root); |
| 382 | 504 | if (!f.has_build_zig) try checkBuildFileExistence(f); |
| 383 | | if (!f.job_queue.recursive) return; |
| 505 | if (!job_queue.recursive) return; |
| 384 | 506 | return queueJobsForDeps(f); |
| 385 | 507 | }, |
| 386 | 508 | .remote => |remote| remote, |
| 387 | 509 | .path_or_url => |path_or_url| { |
| 388 | 510 | if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| { |
| 389 | 511 | var resource: Resource = .{ .dir = dir }; |
| 390 | | return f.runResource(path_or_url, &resource, null); |
| 512 | return f.runResource(path_or_url, &resource, null, false); |
| 391 | 513 | } else |dir_err| { |
| 392 | 514 | var server_header_buffer: [init_resource_buffer_size]u8 = undefined; |
| 393 | 515 | |
| 394 | 516 | const file_err = if (dir_err == error.NotDir) e: { |
| 395 | 517 | if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| { |
| 396 | 518 | var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) }; |
| 397 | | return f.runResource(path_or_url, &resource, null); |
| 519 | return f.runResource(path_or_url, &resource, null, false); |
| 398 | 520 | } else |err| break :e err; |
| 399 | 521 | } else dir_err; |
| 400 | 522 | |
| ... | ... | @@ -406,57 +528,73 @@ pub fn run(f: *Fetch) RunError!void { |
| 406 | 528 | }; |
| 407 | 529 | var resource: Resource = undefined; |
| 408 | 530 | try f.initResource(uri, &resource, &server_header_buffer); |
| 409 | | return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null); |
| 531 | return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false); |
| 410 | 532 | } |
| 411 | 533 | }, |
| 412 | 534 | }; |
| 413 | 535 | |
| 536 | var resource_buffer: [init_resource_buffer_size]u8 = undefined; |
| 537 | |
| 414 | 538 | if (remote.hash) |expected_hash| { |
| 415 | | var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined; |
| 416 | | prefixed_pkg_sub_path_buffer[0] = 'p'; |
| 417 | | prefixed_pkg_sub_path_buffer[1] = fs.path.sep; |
| 418 | | const hash_slice = expected_hash.toSlice(); |
| 419 | | @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice); |
| 420 | | const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len]; |
| 421 | | const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0; |
| 422 | | const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..]; |
| 423 | | if (cache_root.handle.access(io, pkg_sub_path, .{})) |_| { |
| 539 | const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice()); |
| 540 | if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| { |
| 424 | 541 | assert(f.lazy_status != .unavailable); |
| 425 | | f.package_root = .{ |
| 426 | | .root_dir = cache_root, |
| 427 | | .sub_path = try arena.dupe(u8, pkg_sub_path), |
| 428 | | }; |
| 542 | f.package_root = package_root; |
| 429 | 543 | try loadManifest(f, f.package_root); |
| 430 | 544 | try checkBuildFileExistence(f); |
| 431 | | if (!f.job_queue.recursive) return; |
| 545 | if (!job_queue.recursive) return; |
| 432 | 546 | return queueJobsForDeps(f); |
| 433 | 547 | } else |err| switch (err) { |
| 434 | 548 | error.FileNotFound => { |
| 435 | | switch (f.lazy_status) { |
| 436 | | .eager => {}, |
| 437 | | .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) { |
| 438 | | f.lazy_status = .unavailable; |
| 439 | | return; |
| 440 | | }, |
| 441 | | .unavailable => unreachable, |
| 442 | | } |
| 443 | | if (f.job_queue.read_only) return f.fail( |
| 549 | log.debug("FileNotFound: {f}", .{package_root}); |
| 550 | if (job_queue.read_only) return f.fail( |
| 444 | 551 | f.name_tok, |
| 445 | | try eb.printString("package not found at '{f}{s}'", .{ |
| 446 | | cache_root, pkg_sub_path, |
| 447 | | }), |
| 552 | try eb.printString("package not found at '{f}'", .{package_root}), |
| 448 | 553 | ); |
| 449 | 554 | }, |
| 555 | error.Canceled => |e| return e, |
| 556 | else => |e| { |
| 557 | try eb.addRootErrorMessage(.{ |
| 558 | .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{ |
| 559 | package_root, e, |
| 560 | }), |
| 561 | }); |
| 562 | return error.FetchFailed; |
| 563 | }, |
| 564 | } |
| 565 | |
| 566 | // Check global cache before remote fetch. |
| 567 | const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()}); |
| 568 | const cached_tarball_path: Cache.Path = .{ |
| 569 | .root_dir = job_queue.global_cache, |
| 570 | .sub_path = cached_tarball_sub_path, |
| 571 | }; |
| 572 | if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) |file| { |
| 573 | log.debug("found global cached tarball {f}", .{cached_tarball_path}); |
| 574 | var resource: Resource = .{ .file = file.reader(io, &resource_buffer) }; |
| 575 | return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true); |
| 576 | } else |err| switch (err) { |
| 577 | error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}), |
| 578 | error.Canceled => |e| return e, |
| 450 | 579 | else => |e| { |
| 451 | 580 | try eb.addRootErrorMessage(.{ |
| 452 | | .msg = try eb.printString("unable to open global package cache directory '{f}{s}': {s}", .{ |
| 453 | | cache_root, pkg_sub_path, @errorName(e), |
| 581 | .msg = try eb.printString("unable to open globally cached package {f}: {t}", .{ |
| 582 | cached_tarball_path, e, |
| 454 | 583 | }), |
| 455 | 584 | }); |
| 456 | 585 | return error.FetchFailed; |
| 457 | 586 | }, |
| 458 | 587 | } |
| 459 | | } else if (f.job_queue.read_only) { |
| 588 | |
| 589 | switch (f.lazy_status) { |
| 590 | .eager => {}, |
| 591 | .available => if (!job_queue.unlazy_set.contains(expected_hash)) { |
| 592 | f.lazy_status = .unavailable; |
| 593 | return; |
| 594 | }, |
| 595 | .unavailable => unreachable, |
| 596 | } |
| 597 | } else if (job_queue.read_only) { |
| 460 | 598 | try eb.addRootErrorMessage(.{ |
| 461 | 599 | .msg = try eb.addString("dependency is missing hash field"), |
| 462 | 600 | .src_loc = try f.srcLoc(f.location_tok), |
| ... | ... | @@ -465,15 +603,13 @@ pub fn run(f: *Fetch) RunError!void { |
| 465 | 603 | } |
| 466 | 604 | |
| 467 | 605 | // Fetch and unpack the remote into a temporary directory. |
| 468 | | |
| 469 | 606 | const uri = std.Uri.parse(remote.url) catch |err| return f.fail( |
| 470 | 607 | f.location_tok, |
| 471 | | try eb.printString("invalid URI: {s}", .{@errorName(err)}), |
| 608 | try eb.printString("invalid URI: {t}", .{err}), |
| 472 | 609 | ); |
| 473 | | var buffer: [init_resource_buffer_size]u8 = undefined; |
| 474 | 610 | var resource: Resource = undefined; |
| 475 | | try f.initResource(uri, &resource, &buffer); |
| 476 | | return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash); |
| 611 | try f.initResource(uri, &resource, &resource_buffer); |
| 612 | return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false); |
| 477 | 613 | } |
| 478 | 614 | |
| 479 | 615 | pub fn deinit(f: *Fetch) void { |
| ... | ... | @@ -487,30 +623,35 @@ fn runResource( |
| 487 | 623 | uri_path: []const u8, |
| 488 | 624 | resource: *Resource, |
| 489 | 625 | remote_hash: ?Package.Hash, |
| 626 | disable_recompress: bool, |
| 490 | 627 | ) RunError!void { |
| 491 | | const io = f.job_queue.io; |
| 628 | const job_queue = f.job_queue; |
| 629 | assert(!job_queue.read_only); |
| 630 | |
| 631 | const io = job_queue.io; |
| 492 | 632 | defer resource.deinit(io); |
| 633 | |
| 493 | 634 | const arena = f.arena.allocator(); |
| 494 | 635 | const eb = &f.error_bundle; |
| 495 | 636 | const s = fs.path.sep_str; |
| 496 | | const cache_root = f.job_queue.global_cache; |
| 637 | const local_cache_root = job_queue.local_cache; |
| 497 | 638 | const rand_int = r: { |
| 498 | 639 | var x: u64 = undefined; |
| 499 | 640 | io.random(@ptrCast(&x)); |
| 500 | 641 | break :r x; |
| 501 | 642 | }; |
| 502 | 643 | const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int); |
| 644 | const tmp_directory_path = try local_cache_root.join(arena, tmp_dir_sub_path); |
| 503 | 645 | |
| 504 | 646 | const package_sub_path = blk: { |
| 505 | | const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); |
| 506 | 647 | var tmp_directory: Cache.Directory = .{ |
| 507 | | .path = tmp_directory_path, |
| 648 | .path = tmp_directory_path.sub_path, |
| 508 | 649 | .handle = handle: { |
| 509 | | const dir = cache_root.handle.createDirPathOpen(io, tmp_dir_sub_path, .{ |
| 650 | const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{ |
| 510 | 651 | .open_options = .{ .iterate = true }, |
| 511 | 652 | }) catch |err| { |
| 512 | 653 | try eb.addRootErrorMessage(.{ |
| 513 | | .msg = try eb.printString("unable to create temporary directory '{s}': {t}", .{ |
| 654 | .msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{ |
| 514 | 655 | tmp_directory_path, err, |
| 515 | 656 | }), |
| 516 | 657 | }); |
| ... | ... | @@ -524,16 +665,7 @@ fn runResource( |
| 524 | 665 | // Fetch and unpack a resource into a temporary directory. |
| 525 | 666 | var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory); |
| 526 | 667 | |
| 527 | | var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; |
| 528 | | |
| 529 | | // Apply btrfs workaround if needed. Reopen tmp_directory. |
| 530 | | if (native_os == .linux and f.job_queue.work_around_btrfs_bug) { |
| 531 | | // https://github.com/ziglang/zig/issues/17095 |
| 532 | | pkg_path.root_dir.handle.close(io); |
| 533 | | pkg_path.root_dir.handle = cache_root.handle.createDirPathOpen(io, tmp_dir_sub_path, .{ |
| 534 | | .open_options = .{ .iterate = true }, |
| 535 | | }) catch @panic("btrfs workaround failed"); |
| 536 | | } |
| 668 | const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir }; |
| 537 | 669 | |
| 538 | 670 | // Load, parse, and validate the unpacked build.zig.zon file. It is allowed |
| 539 | 671 | // for the file to be missing, in which case this fetched package is |
| ... | ... | @@ -555,36 +687,40 @@ fn runResource( |
| 555 | 687 | // directory. |
| 556 | 688 | f.computed_hash = try computeHash(f, pkg_path, filter); |
| 557 | 689 | |
| 558 | | break :blk if (unpack_result.root_dir.len > 0) |
| 559 | | try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) |
| 560 | | else |
| 561 | | tmp_dir_sub_path; |
| 690 | if (unpack_result.root_dir.len > 0) |
| 691 | break :blk try tmp_directory_path.join(arena, unpack_result.root_dir); |
| 692 | |
| 693 | break :blk tmp_directory_path; |
| 562 | 694 | }; |
| 563 | 695 | |
| 564 | 696 | const computed_package_hash = computedPackageHash(f); |
| 565 | 697 | |
| 566 | | // Rename the temporary directory into the global zig package cache |
| 567 | | // directory. If the hash already exists, delete the temporary directory |
| 568 | | // and leave the zig package cache directory untouched as it may be in use |
| 569 | | // by the system. This is done even if the hash is invalid, in case the |
| 570 | | // package with the different hash is used in the future. |
| 571 | | |
| 572 | | f.package_root = .{ |
| 573 | | .root_dir = cache_root, |
| 574 | | .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}), |
| 575 | | }; |
| 576 | | renameTmpIntoCache(io, cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { |
| 577 | | const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); |
| 578 | | const dest = try cache_root.join(arena, &.{f.package_root.sub_path}); |
| 698 | // Rename the temporary directory into the local zig package directory. If |
| 699 | // the hash already exists, delete the temporary directory and leave the |
| 700 | // zig package directory untouched as it may be in use. This is done even |
| 701 | // if the hash is invalid, in case the package with the different hash is |
| 702 | // used in the future. |
| 703 | f.package_root = try job_queue.root_pkg_path.join(arena, computed_package_hash.toSlice()); |
| 704 | renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| { |
| 579 | 705 | try eb.addRootErrorMessage(.{ .msg = try eb.printString( |
| 580 | | "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", |
| 581 | | .{ src, dest, @errorName(err) }, |
| 706 | "unable to rename temporary directory {f} into package cache directory {f}: {t}", |
| 707 | .{ package_sub_path, f.package_root, err }, |
| 582 | 708 | ) }); |
| 583 | 709 | return error.FetchFailed; |
| 584 | 710 | }; |
| 711 | |
| 712 | if (!disable_recompress) { |
| 713 | // Spin off a task to recompress the tarball, with filtered files deleted, into |
| 714 | // the global cache. |
| 715 | job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash }); |
| 716 | } |
| 717 | |
| 585 | 718 | // Remove temporary directory root if not already renamed to global cache. |
| 586 | | if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) { |
| 587 | | cache_root.handle.deleteDir(io, tmp_dir_sub_path) catch {}; |
| 719 | if (!package_sub_path.eql(tmp_directory_path)) { |
| 720 | tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) { |
| 721 | error.Canceled => |e| return e, |
| 722 | else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }), |
| 723 | }; |
| 588 | 724 | } |
| 589 | 725 | |
| 590 | 726 | // Validate the computed hash against the expected hash. If invalid, this |
| ... | ... | @@ -624,7 +760,7 @@ fn runResource( |
| 624 | 760 | |
| 625 | 761 | // Spawn a new fetch job for each dependency in the manifest file. Use |
| 626 | 762 | // a mutex and a hash map so that redundant jobs do not get queued up. |
| 627 | | if (!f.job_queue.recursive) return; |
| 763 | if (!job_queue.recursive) return; |
| 628 | 764 | return queueJobsForDeps(f); |
| 629 | 765 | } |
| 630 | 766 | |
| ... | ... | @@ -651,8 +787,8 @@ fn checkBuildFileExistence(f: *Fetch) RunError!void { |
| 651 | 787 | error.FileNotFound => {}, |
| 652 | 788 | else => |e| { |
| 653 | 789 | try eb.addRootErrorMessage(.{ |
| 654 | | .msg = try eb.printString("unable to access '{f}{s}': {s}", .{ |
| 655 | | f.package_root, Package.build_zig_basename, @errorName(e), |
| 790 | .msg = try eb.printString("unable to access '{f}{s}': {t}", .{ |
| 791 | f.package_root, Package.build_zig_basename, e, |
| 656 | 792 | }), |
| 657 | 793 | }); |
| 658 | 794 | return error.FetchFailed; |
| ... | ... | @@ -677,9 +813,7 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { |
| 677 | 813 | else => |e| { |
| 678 | 814 | const file_path = try pkg_root.join(arena, Manifest.basename); |
| 679 | 815 | try eb.addRootErrorMessage(.{ |
| 680 | | .msg = try eb.printString("unable to load package manifest '{f}': {s}", .{ |
| 681 | | file_path, @errorName(e), |
| 682 | | }), |
| 816 | .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ file_path, e }), |
| 683 | 817 | }); |
| 684 | 818 | return error.FetchFailed; |
| 685 | 819 | }, |
| ... | ... | @@ -698,8 +832,6 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void { |
| 698 | 832 | |
| 699 | 833 | f.manifest = try Manifest.parse(arena, ast.*, rng.interface(), .{ |
| 700 | 834 | .allow_missing_paths_field = f.allow_missing_paths_field, |
| 701 | | .allow_missing_fingerprint = f.allow_missing_fingerprint, |
| 702 | | .allow_name_string = f.allow_name_string, |
| 703 | 835 | }); |
| 704 | 836 | const manifest = &f.manifest.?; |
| 705 | 837 | |
| ... | ... | @@ -817,8 +949,6 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { |
| 817 | 949 | .job_queue = f.job_queue, |
| 818 | 950 | .omit_missing_hash_error = false, |
| 819 | 951 | .allow_missing_paths_field = true, |
| 820 | | .allow_missing_fingerprint = true, |
| 821 | | .allow_name_string = true, |
| 822 | 952 | .use_latest_commit = false, |
| 823 | 953 | |
| 824 | 954 | .package_root = undefined, |
| ... | ... | @@ -1463,14 +1593,20 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void |
| 1463 | 1593 | } |
| 1464 | 1594 | } |
| 1465 | 1595 | |
| 1466 | | pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void { |
| 1467 | | assert(dest_dir_sub_path[1] == fs.path.sep); |
| 1596 | pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void { |
| 1468 | 1597 | var handled_missing_dir = false; |
| 1469 | 1598 | while (true) { |
| 1470 | | cache_dir.rename(tmp_dir_sub_path, cache_dir, dest_dir_sub_path, io) catch |err| switch (err) { |
| 1599 | Io.Dir.rename( |
| 1600 | tmp_path.root_dir.handle, |
| 1601 | tmp_path.sub_path, |
| 1602 | dest_path.root_dir.handle, |
| 1603 | dest_path.sub_path, |
| 1604 | io, |
| 1605 | ) catch |err| switch (err) { |
| 1471 | 1606 | error.FileNotFound => { |
| 1472 | 1607 | if (handled_missing_dir) return err; |
| 1473 | | cache_dir.createDir(io, dest_dir_sub_path[0..1], .default_dir) catch |mkd_err| switch (mkd_err) { |
| 1608 | const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?; |
| 1609 | dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) { |
| 1474 | 1610 | error.PathAlreadyExists => handled_missing_dir = true, |
| 1475 | 1611 | else => |e| return e, |
| 1476 | 1612 | }; |
| ... | ... | @@ -1478,9 +1614,11 @@ pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u |
| 1478 | 1614 | }, |
| 1479 | 1615 | error.DirNotEmpty, error.AccessDenied => { |
| 1480 | 1616 | // Package has been already downloaded and may already be in use on the system. |
| 1481 | | cache_dir.deleteTree(io, tmp_dir_sub_path) catch { |
| 1617 | tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) { |
| 1618 | error.Canceled => |e| return e, |
| 1482 | 1619 | // Garbage files leftover in zig-cache/tmp/ is, as they say |
| 1483 | 1620 | // on Star Trek, "operating within normal parameters". |
| 1621 | else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }), |
| 1484 | 1622 | }; |
| 1485 | 1623 | }, |
| 1486 | 1624 | else => |e| return e, |
| ... | ... | @@ -2064,130 +2202,6 @@ const UnpackResult = struct { |
| 2064 | 2202 | } |
| 2065 | 2203 | }; |
| 2066 | 2204 | |
| 2067 | | test "tarball with duplicate paths" { |
| 2068 | | // This tarball has duplicate path 'dir1/file1' to simulate case sensitve |
| 2069 | | // file system on any file sytstem. |
| 2070 | | // |
| 2071 | | // duplicate_paths/ |
| 2072 | | // duplicate_paths/dir1/ |
| 2073 | | // duplicate_paths/dir1/file1 |
| 2074 | | // duplicate_paths/dir1/file1 |
| 2075 | | // duplicate_paths/build.zig.zon |
| 2076 | | // duplicate_paths/src/ |
| 2077 | | // duplicate_paths/src/main.zig |
| 2078 | | // duplicate_paths/src/root.zig |
| 2079 | | // duplicate_paths/build.zig |
| 2080 | | // |
| 2081 | | |
| 2082 | | const gpa = std.testing.allocator; |
| 2083 | | const io = std.testing.io; |
| 2084 | | var tmp = std.testing.tmpDir(.{}); |
| 2085 | | defer tmp.cleanup(); |
| 2086 | | |
| 2087 | | const tarball_name = "duplicate_paths.tar.gz"; |
| 2088 | | try saveEmbedFile(io, tarball_name, tmp.dir); |
| 2089 | | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| 2090 | | defer gpa.free(tarball_path); |
| 2091 | | |
| 2092 | | // Run tarball fetch, expect to fail |
| 2093 | | var fb: TestFetchBuilder = undefined; |
| 2094 | | var fetch = try fb.build(gpa, io, tmp.dir, tarball_path); |
| 2095 | | defer fb.deinit(); |
| 2096 | | try std.testing.expectError(error.FetchFailed, fetch.run()); |
| 2097 | | |
| 2098 | | try fb.expectFetchErrors(1, |
| 2099 | | \\error: unable to unpack tarball |
| 2100 | | \\ note: unable to create file 'dir1/file1': PathAlreadyExists |
| 2101 | | \\ |
| 2102 | | ); |
| 2103 | | } |
| 2104 | | |
| 2105 | | test "tarball with excluded duplicate paths" { |
| 2106 | | // Same as previous tarball but has build.zig.zon wich excludes 'dir1'. |
| 2107 | | // |
| 2108 | | // .paths = .{ |
| 2109 | | // "build.zig", |
| 2110 | | // "build.zig.zon", |
| 2111 | | // "src", |
| 2112 | | // } |
| 2113 | | // |
| 2114 | | |
| 2115 | | const gpa = std.testing.allocator; |
| 2116 | | const io = std.testing.io; |
| 2117 | | var tmp = std.testing.tmpDir(.{}); |
| 2118 | | defer tmp.cleanup(); |
| 2119 | | |
| 2120 | | const tarball_name = "duplicate_paths_excluded.tar.gz"; |
| 2121 | | try saveEmbedFile(io, tarball_name, tmp.dir); |
| 2122 | | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| 2123 | | defer gpa.free(tarball_path); |
| 2124 | | |
| 2125 | | // Run tarball fetch, should succeed |
| 2126 | | var fb: TestFetchBuilder = undefined; |
| 2127 | | var fetch = try fb.build(gpa, io, tmp.dir, tarball_path); |
| 2128 | | defer fb.deinit(); |
| 2129 | | try fetch.run(); |
| 2130 | | |
| 2131 | | const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); |
| 2132 | | try std.testing.expectEqualStrings( |
| 2133 | | "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", |
| 2134 | | &hex_digest, |
| 2135 | | ); |
| 2136 | | |
| 2137 | | const expected_files: []const []const u8 = &.{ |
| 2138 | | "build.zig", |
| 2139 | | "build.zig.zon", |
| 2140 | | "src/main.zig", |
| 2141 | | "src/root.zig", |
| 2142 | | }; |
| 2143 | | try fb.expectPackageFiles(expected_files); |
| 2144 | | } |
| 2145 | | |
| 2146 | | test "tarball without root folder" { |
| 2147 | | // Tarball with root folder. Manifest excludes dir1 and dir2. |
| 2148 | | // |
| 2149 | | // build.zig |
| 2150 | | // build.zig.zon |
| 2151 | | // dir1/ |
| 2152 | | // dir1/file2 |
| 2153 | | // dir1/file1 |
| 2154 | | // dir2/ |
| 2155 | | // dir2/file2 |
| 2156 | | // src/ |
| 2157 | | // src/main.zig |
| 2158 | | // |
| 2159 | | |
| 2160 | | const gpa = std.testing.allocator; |
| 2161 | | const io = std.testing.io; |
| 2162 | | |
| 2163 | | var tmp = std.testing.tmpDir(.{}); |
| 2164 | | defer tmp.cleanup(); |
| 2165 | | |
| 2166 | | const tarball_name = "no_root.tar.gz"; |
| 2167 | | try saveEmbedFile(io, tarball_name, tmp.dir); |
| 2168 | | const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name }); |
| 2169 | | defer gpa.free(tarball_path); |
| 2170 | | |
| 2171 | | // Run tarball fetch, should succeed |
| 2172 | | var fb: TestFetchBuilder = undefined; |
| 2173 | | var fetch = try fb.build(gpa, io, tmp.dir, tarball_path); |
| 2174 | | defer fb.deinit(); |
| 2175 | | try fetch.run(); |
| 2176 | | |
| 2177 | | const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); |
| 2178 | | try std.testing.expectEqualStrings( |
| 2179 | | "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", |
| 2180 | | &hex_digest, |
| 2181 | | ); |
| 2182 | | |
| 2183 | | const expected_files: []const []const u8 = &.{ |
| 2184 | | "build.zig", |
| 2185 | | "build.zig.zon", |
| 2186 | | "src/main.zig", |
| 2187 | | }; |
| 2188 | | try fb.expectPackageFiles(expected_files); |
| 2189 | | } |
| 2190 | | |
| 2191 | 2205 | test "set executable bit based on file content" { |
| 2192 | 2206 | if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest; |
| 2193 | 2207 | const gpa = std.testing.allocator; |
| ... | ... | @@ -2254,6 +2268,7 @@ fn saveEmbedFile(io: Io, comptime tarball_name: []const u8, dir: Io.Dir) !void { |
| 2254 | 2268 | const TestFetchBuilder = struct { |
| 2255 | 2269 | http_client: std.http.Client, |
| 2256 | 2270 | global_cache_directory: Cache.Directory, |
| 2271 | local_cache_path: Cache.Path, |
| 2257 | 2272 | job_queue: Fetch.JobQueue, |
| 2258 | 2273 | fetch: Fetch, |
| 2259 | 2274 | |
| ... | ... | @@ -2264,20 +2279,30 @@ const TestFetchBuilder = struct { |
| 2264 | 2279 | cache_parent_dir: std.Io.Dir, |
| 2265 | 2280 | path_or_url: []const u8, |
| 2266 | 2281 | ) !*Fetch { |
| 2267 | | const cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{}); |
| 2282 | const global_cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{}); |
| 2283 | const package_root_dir = try cache_parent_dir.createDirPathOpen(io, "local-project-root", .{}); |
| 2268 | 2284 | |
| 2269 | 2285 | self.http_client = .{ .allocator = allocator, .io = io }; |
| 2270 | | self.global_cache_directory = .{ .handle = cache_dir, .path = null }; |
| 2286 | self.global_cache_directory = .{ .handle = global_cache_dir, .path = "zig-global-cache" }; |
| 2287 | self.local_cache_path = .{ |
| 2288 | .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" }, |
| 2289 | .sub_path = ".zig-cache", |
| 2290 | }; |
| 2271 | 2291 | |
| 2272 | 2292 | self.job_queue = .{ |
| 2273 | 2293 | .io = io, |
| 2274 | 2294 | .http_client = &self.http_client, |
| 2275 | 2295 | .global_cache = self.global_cache_directory, |
| 2296 | .local_cache = self.local_cache_path, |
| 2297 | .root_pkg_path = .{ |
| 2298 | .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" }, |
| 2299 | .sub_path = "zig-pkg", |
| 2300 | }, |
| 2276 | 2301 | .recursive = false, |
| 2277 | 2302 | .read_only = false, |
| 2278 | 2303 | .debug_hash = false, |
| 2279 | | .work_around_btrfs_bug = false, |
| 2280 | 2304 | .mode = .needed, |
| 2305 | .prog_node = std.Progress.Node.none, |
| 2281 | 2306 | }; |
| 2282 | 2307 | |
| 2283 | 2308 | self.fetch = .{ |
| ... | ... | @@ -2287,14 +2312,12 @@ const TestFetchBuilder = struct { |
| 2287 | 2312 | .hash_tok = .none, |
| 2288 | 2313 | .name_tok = 0, |
| 2289 | 2314 | .lazy_status = .eager, |
| 2290 | | .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } }, |
| 2315 | .parent_package_root = .{ .root_dir = .{ .handle = package_root_dir, .path = null } }, |
| 2291 | 2316 | .parent_manifest_ast = null, |
| 2292 | 2317 | .prog_node = std.Progress.Node.none, |
| 2293 | 2318 | .job_queue = &self.job_queue, |
| 2294 | 2319 | .omit_missing_hash_error = true, |
| 2295 | 2320 | .allow_missing_paths_field = false, |
| 2296 | | .allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz |
| 2297 | | .allow_name_string = true, // so we can keep using the old testdata .tar.gz |
| 2298 | 2321 | .use_latest_commit = true, |
| 2299 | 2322 | |
| 2300 | 2323 | .package_root = undefined, |
Andrew Kelley <andrew@ziglang.org> 2026-02-06 09:41:28+01:00
lib