| ... | ... | @@ -228,6 +228,7 @@ pub fn fetchAndAddDependencies( |
| 228 | 228 | name_prefix: []const u8, |
| 229 | 229 | error_bundle: *std.zig.ErrorBundle.Wip, |
| 230 | 230 | all_modules: *AllModules, |
| 231 | root_prog_node: *std.Progress.Node, |
| 231 | 232 | ) !void { |
| 232 | 233 | const max_bytes = 10 * 1024 * 1024; |
| 233 | 234 | const gpa = thread_pool.allocator; |
| ... | ... | @@ -272,6 +273,17 @@ pub fn fetchAndAddDependencies( |
| 272 | 273 | .error_bundle = error_bundle, |
| 273 | 274 | }; |
| 274 | 275 | |
| 276 | for (manifest.dependencies.values()) |dep| { |
| 277 | // If the hash is invalid, let errors happen later |
| 278 | // We only want to add these for progress reporting |
| 279 | const hash = dep.hash orelse continue; |
| 280 | if (hash.len != hex_multihash_len) continue; |
| 281 | const gop = try all_modules.getOrPut(gpa, hash[0..hex_multihash_len].*); |
| 282 | if (!gop.found_existing) gop.value_ptr.* = null; |
| 283 | } |
| 284 | |
| 285 | root_prog_node.setEstimatedTotalItems(all_modules.count()); |
| 286 | |
| 275 | 287 | const deps_list = manifest.dependencies.values(); |
| 276 | 288 | for (manifest.dependencies.keys(), 0..) |name, i| { |
| 277 | 289 | const dep = deps_list[i]; |
| ... | ... | @@ -288,6 +300,7 @@ pub fn fetchAndAddDependencies( |
| 288 | 300 | build_roots_source, |
| 289 | 301 | fqn, |
| 290 | 302 | all_modules, |
| 303 | root_prog_node, |
| 291 | 304 | ); |
| 292 | 305 | |
| 293 | 306 | if (!sub.found_existing) { |
| ... | ... | @@ -304,6 +317,7 @@ pub fn fetchAndAddDependencies( |
| 304 | 317 | sub_prefix, |
| 305 | 318 | error_bundle, |
| 306 | 319 | all_modules, |
| 320 | root_prog_node, |
| 307 | 321 | ); |
| 308 | 322 | } |
| 309 | 323 | |
| ... | ... | @@ -404,7 +418,51 @@ const Report = struct { |
| 404 | 418 | const hex_multihash_len = 2 * Manifest.multihash_len; |
| 405 | 419 | const MultiHashHexDigest = [hex_multihash_len]u8; |
| 406 | 420 | /// This is to avoid creating multiple modules for the same build.zig file. |
| 407 | | pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, *Package); |
| 421 | /// If the value is `null`, the package is a known dependency, but has not yet |
| 422 | /// been fetched. |
| 423 | pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?*Package); |
| 424 | |
| 425 | fn ProgressReader(comptime ReaderType: type) type { |
| 426 | return struct { |
| 427 | child_reader: ReaderType, |
| 428 | bytes_read: u64 = 0, |
| 429 | prog_node: *std.Progress.Node, |
| 430 | unit: enum { |
| 431 | kib, |
| 432 | mib, |
| 433 | any, |
| 434 | }, |
| 435 | |
| 436 | pub const Error = ReaderType.Error; |
| 437 | pub const Reader = std.io.Reader(*@This(), Error, read); |
| 438 | |
| 439 | pub fn read(self: *@This(), buf: []u8) Error!usize { |
| 440 | const amt = try self.child_reader.read(buf); |
| 441 | self.bytes_read += amt; |
| 442 | const kib = self.bytes_read / 1024; |
| 443 | const mib = kib / 1024; |
| 444 | switch (self.unit) { |
| 445 | .kib => self.prog_node.setCompletedItems(@intCast(kib)), |
| 446 | .mib => self.prog_node.setCompletedItems(@intCast(mib)), |
| 447 | .any => { |
| 448 | if (mib > 0) { |
| 449 | self.prog_node.setUnit("MiB"); |
| 450 | self.prog_node.setCompletedItems(@intCast(mib)); |
| 451 | } else { |
| 452 | self.prog_node.setUnit("KiB"); |
| 453 | self.prog_node.setCompletedItems(@intCast(kib)); |
| 454 | } |
| 455 | }, |
| 456 | } |
| 457 | self.prog_node.context.maybeRefresh(); |
| 458 | return amt; |
| 459 | } |
| 460 | |
| 461 | pub fn reader(self: *@This()) Reader { |
| 462 | return .{ .context = self }; |
| 463 | } |
| 464 | }; |
| 465 | } |
| 408 | 466 | |
| 409 | 467 | fn fetchAndUnpack( |
| 410 | 468 | thread_pool: *ThreadPool, |
| ... | ... | @@ -415,6 +473,7 @@ fn fetchAndUnpack( |
| 415 | 473 | build_roots_source: *std.ArrayList(u8), |
| 416 | 474 | fqn: []const u8, |
| 417 | 475 | all_modules: *AllModules, |
| 476 | root_prog_node: *std.Progress.Node, |
| 418 | 477 | ) !struct { mod: *Package, found_existing: bool } { |
| 419 | 478 | const gpa = http_client.allocator; |
| 420 | 479 | const s = fs.path.sep_str; |
| ... | ... | @@ -442,13 +501,17 @@ fn fetchAndUnpack( |
| 442 | 501 | // so we must detect if a module has been created for this package and reuse it. |
| 443 | 502 | const gop = try all_modules.getOrPut(gpa, hex_digest.*); |
| 444 | 503 | if (gop.found_existing) { |
| 445 | | gpa.free(build_root); |
| 446 | | return .{ |
| 447 | | .mod = gop.value_ptr.*, |
| 448 | | .found_existing = true, |
| 449 | | }; |
| 504 | if (gop.value_ptr.*) |mod| { |
| 505 | gpa.free(build_root); |
| 506 | return .{ |
| 507 | .mod = mod, |
| 508 | .found_existing = true, |
| 509 | }; |
| 510 | } |
| 450 | 511 | } |
| 451 | 512 | |
| 513 | root_prog_node.completeOne(); |
| 514 | |
| 452 | 515 | const ptr = try gpa.create(Package); |
| 453 | 516 | errdefer gpa.destroy(ptr); |
| 454 | 517 | |
| ... | ... | @@ -471,6 +534,11 @@ fn fetchAndUnpack( |
| 471 | 534 | }; |
| 472 | 535 | } |
| 473 | 536 | |
| 537 | var pkg_prog_node = root_prog_node.start(fqn, 0); |
| 538 | defer pkg_prog_node.end(); |
| 539 | pkg_prog_node.activate(); |
| 540 | pkg_prog_node.context.refresh(); |
| 541 | |
| 474 | 542 | const uri = try std.Uri.parse(dep.url); |
| 475 | 543 | |
| 476 | 544 | const rand_int = std.crypto.random.int(u64); |
| ... | ... | @@ -510,29 +578,53 @@ fn fetchAndUnpack( |
| 510 | 578 | const content_type = req.response.headers.getFirstValue("Content-Type") orelse |
| 511 | 579 | return report.fail(dep.url_tok, "Missing 'Content-Type' header", .{}); |
| 512 | 580 | |
| 581 | var prog_reader: ProgressReader(std.http.Client.Request.Reader) = .{ |
| 582 | .child_reader = req.reader(), |
| 583 | .prog_node = &pkg_prog_node, |
| 584 | .unit = if (req.response.content_length) |content_length| unit: { |
| 585 | const kib = content_length / 1024; |
| 586 | const mib = kib / 1024; |
| 587 | if (mib > 0) { |
| 588 | pkg_prog_node.setEstimatedTotalItems(@intCast(mib)); |
| 589 | pkg_prog_node.setUnit("MiB"); |
| 590 | break :unit .mib; |
| 591 | } else { |
| 592 | pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib))); |
| 593 | pkg_prog_node.setUnit("KiB"); |
| 594 | break :unit .kib; |
| 595 | } |
| 596 | } else .any, |
| 597 | }; |
| 598 | pkg_prog_node.context.refresh(); |
| 599 | |
| 513 | 600 | if (ascii.eqlIgnoreCase(content_type, "application/gzip") or |
| 514 | 601 | ascii.eqlIgnoreCase(content_type, "application/x-gzip") or |
| 515 | 602 | ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) |
| 516 | 603 | { |
| 517 | 604 | // I observed the gzip stream to read 1 byte at a time, so I am using a |
| 518 | 605 | // buffered reader on the front of it. |
| 519 | | try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.gzip); |
| 606 | try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip); |
| 520 | 607 | } else if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) { |
| 521 | 608 | // I have not checked what buffer sizes the xz decompression implementation uses |
| 522 | 609 | // by default, so the same logic applies for buffering the reader as for gzip. |
| 523 | | try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.xz); |
| 610 | try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.xz); |
| 524 | 611 | } else if (ascii.eqlIgnoreCase(content_type, "application/octet-stream")) { |
| 525 | 612 | // support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz |
| 526 | 613 | // whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"' |
| 527 | 614 | const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse |
| 528 | 615 | return report.fail(dep.url_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{}); |
| 529 | 616 | if (isTarAttachment(content_disposition)) { |
| 530 | | try unpackTarball(gpa, &req, tmp_directory.handle, std.compress.gzip); |
| 617 | try unpackTarball(gpa, prog_reader.reader(), tmp_directory.handle, std.compress.gzip); |
| 531 | 618 | } else return report.fail(dep.url_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition}); |
| 532 | 619 | } else { |
| 533 | 620 | return report.fail(dep.url_tok, "Unsupported 'Content-Type' header value: '{s}'", .{content_type}); |
| 534 | 621 | } |
| 535 | 622 | |
| 623 | // Download completed - stop showing downloaded amount as progress |
| 624 | pkg_prog_node.setEstimatedTotalItems(0); |
| 625 | pkg_prog_node.setCompletedItems(0); |
| 626 | pkg_prog_node.context.refresh(); |
| 627 | |
| 536 | 628 | // TODO: delete files not included in the package prior to computing the package hash. |
| 537 | 629 | // for example, if the ini file has directives to include/not include certain files, |
| 538 | 630 | // apply those rules directly to the filesystem right here. This ensures that files |
| ... | ... | @@ -591,11 +683,11 @@ fn fetchAndUnpack( |
| 591 | 683 | |
| 592 | 684 | fn unpackTarball( |
| 593 | 685 | gpa: Allocator, |
| 594 | | req: *std.http.Client.Request, |
| 686 | req_reader: anytype, |
| 595 | 687 | out_dir: fs.Dir, |
| 596 | 688 | comptime compression: type, |
| 597 | 689 | ) !void { |
| 598 | | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req.reader()); |
| 690 | var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, req_reader); |
| 599 | 691 | |
| 600 | 692 | var decompress = try compression.decompress(gpa, br.reader()); |
| 601 | 693 | defer decompress.deinit(); |
Matthew Lugg <mlugg@mlugg.co.uk> 2023-07-25 14:33:19+01:00
Commit is signed but in an unrecognized format.