authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-07 17:50:07-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-07 19:55:40-07:00
logaf2ac24333a7de1abecc784cc1bc7e2ef005c873
treea4d41b3b6bd092b9384a7cb2dac2bf50b61e4f4c
parent5ce8e9325b7aa15cbcc77221fc7075b6c46619cc

Fetch: handle compressed git+http

4 files changed, 87 insertions(+), 90 deletions(-)

lib/std/http.zig+26-18
......@@ -292,6 +292,14 @@ pub const ContentEncoding = enum {
292292 });
293293 return map.get(s);
294294 }
295
296 pub fn minBufferCapacity(ce: ContentEncoding) usize {
297 return switch (ce) {
298 .zstd => std.compress.zstd.default_window_len,
299 .gzip, .deflate => std.compress.flate.max_window_len,
300 .compress, .identity => 0,
301 };
302 }
295303};
296304
297305pub const Connection = enum {
......@@ -464,8 +472,8 @@ pub const Reader = struct {
464472 transfer_encoding: TransferEncoding,
465473 content_length: ?u64,
466474 content_encoding: ContentEncoding,
467 decompressor: *Decompressor,
468 decompression_buffer: []u8,
475 decompress: *Decompress,
476 decompress_buffer: []u8,
469477 ) *std.Io.Reader {
470478 if (transfer_encoding == .none and content_length == null) {
471479 assert(reader.state == .received_head);
......@@ -475,22 +483,22 @@ pub const Reader = struct {
475483 return reader.in;
476484 },
477485 .deflate => {
478 decompressor.* = .{ .flate = .init(reader.in, .zlib, decompression_buffer) };
479 return &decompressor.flate.reader;
486 decompress.* = .{ .flate = .init(reader.in, .zlib, decompress_buffer) };
487 return &decompress.flate.reader;
480488 },
481489 .gzip => {
482 decompressor.* = .{ .flate = .init(reader.in, .gzip, decompression_buffer) };
483 return &decompressor.flate.reader;
490 decompress.* = .{ .flate = .init(reader.in, .gzip, decompress_buffer) };
491 return &decompress.flate.reader;
484492 },
485493 .zstd => {
486 decompressor.* = .{ .zstd = .init(reader.in, decompression_buffer, .{ .verify_checksum = false }) };
487 return &decompressor.zstd.reader;
494 decompress.* = .{ .zstd = .init(reader.in, decompress_buffer, .{ .verify_checksum = false }) };
495 return &decompress.zstd.reader;
488496 },
489497 .compress => unreachable,
490498 }
491499 }
492500 const transfer_reader = bodyReader(reader, transfer_buffer, transfer_encoding, content_length);
493 return decompressor.init(transfer_reader, decompression_buffer, content_encoding);
501 return decompress.init(transfer_reader, decompress_buffer, content_encoding);
494502 }
495503
496504 fn contentLengthStream(
......@@ -692,33 +700,33 @@ pub const Reader = struct {
692700 }
693701};
694702
695pub const Decompressor = union(enum) {
703pub const Decompress = union(enum) {
696704 flate: std.compress.flate.Decompress,
697705 zstd: std.compress.zstd.Decompress,
698706 none: *std.Io.Reader,
699707
700708 pub fn init(
701 decompressor: *Decompressor,
709 decompress: *Decompress,
702710 transfer_reader: *std.Io.Reader,
703711 buffer: []u8,
704712 content_encoding: ContentEncoding,
705713 ) *std.Io.Reader {
706714 switch (content_encoding) {
707715 .identity => {
708 decompressor.* = .{ .none = transfer_reader };
716 decompress.* = .{ .none = transfer_reader };
709717 return transfer_reader;
710718 },
711719 .deflate => {
712 decompressor.* = .{ .flate = .init(transfer_reader, .zlib, buffer) };
713 return &decompressor.flate.reader;
720 decompress.* = .{ .flate = .init(transfer_reader, .zlib, buffer) };
721 return &decompress.flate.reader;
714722 },
715723 .gzip => {
716 decompressor.* = .{ .flate = .init(transfer_reader, .gzip, buffer) };
717 return &decompressor.flate.reader;
724 decompress.* = .{ .flate = .init(transfer_reader, .gzip, buffer) };
725 return &decompress.flate.reader;
718726 },
719727 .zstd => {
720 decompressor.* = .{ .zstd = .init(transfer_reader, buffer, .{ .verify_checksum = false }) };
721 return &decompressor.zstd.reader;
728 decompress.* = .{ .zstd = .init(transfer_reader, buffer, .{ .verify_checksum = false }) };
729 return &decompress.zstd.reader;
722730 },
723731 .compress => unreachable,
724732 }
lib/std/http/Client.zig+6-6
......@@ -724,8 +724,8 @@ pub const Response = struct {
724724 pub fn readerDecompressing(
725725 response: *Response,
726726 transfer_buffer: []u8,
727 decompressor: *http.Decompressor,
728 decompression_buffer: []u8,
727 decompress: *http.Decompress,
728 decompress_buffer: []u8,
729729 ) *Reader {
730730 response.head.invalidateStrings();
731731 const head = &response.head;
......@@ -734,8 +734,8 @@ pub const Response = struct {
734734 head.transfer_encoding,
735735 head.content_length,
736736 head.content_encoding,
737 decompressor,
738 decompression_buffer,
737 decompress,
738 decompress_buffer,
739739 );
740740 }
741741
......@@ -1797,8 +1797,8 @@ pub fn fetch(client: *Client, options: FetchOptions) FetchError!FetchResult {
17971797 defer if (options.decompress_buffer == null) client.allocator.free(decompress_buffer);
17981798
17991799 var transfer_buffer: [64]u8 = undefined;
1800 var decompressor: http.Decompressor = undefined;
1801 const reader = response.readerDecompressing(&transfer_buffer, &decompressor, decompress_buffer);
1800 var decompress: http.Decompress = undefined;
1801 const reader = response.readerDecompressing(&transfer_buffer, &decompress, decompress_buffer);
18021802
18031803 _ = reader.streamRemaining(response_writer) catch |err| switch (err) {
18041804 error.ReadFailed => return response.bodyErr().?,
src/Package/Fetch.zig+17-11
......@@ -883,7 +883,9 @@ const Resource = union(enum) {
883883 const HttpRequest = struct {
884884 request: std.http.Client.Request,
885885 response: std.http.Client.Response,
886 buffer: []u8,
886 transfer_buffer: []u8,
887 decompress: std.http.Decompress,
888 decompress_buffer: []u8,
887889 };
888890
889891 fn deinit(resource: *Resource) void {
......@@ -892,7 +894,6 @@ const Resource = union(enum) {
892894 .http_request => |*http_request| http_request.request.deinit(),
893895 .git => |*git_resource| {
894896 git_resource.fetch_stream.deinit();
895 git_resource.session.deinit();
896897 },
897898 .dir => |*dir| dir.close(),
898899 }
......@@ -902,7 +903,11 @@ const Resource = union(enum) {
902903 fn reader(resource: *Resource) *std.Io.Reader {
903904 return switch (resource.*) {
904905 .file => |*file_reader| return &file_reader.interface,
905 .http_request => |*http_request| return http_request.response.reader(http_request.buffer),
906 .http_request => |*http_request| return http_request.response.readerDecompressing(
907 http_request.transfer_buffer,
908 &http_request.decompress,
909 http_request.decompress_buffer,
910 ),
906911 .git => |*g| return &g.fetch_stream.reader,
907912 .dir => unreachable,
908913 };
......@@ -971,7 +976,6 @@ const FileType = enum {
971976const init_resource_buffer_size = git.Packet.max_data_length;
972977
973978fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void {
974 const gpa = f.arena.child_allocator;
975979 const arena = f.arena.allocator();
976980 const eb = &f.error_bundle;
977981
......@@ -995,7 +999,9 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
995999 .request = http_client.request(.GET, uri, .{}) catch |err|
9961000 return f.fail(f.location_tok, try eb.printString("unable to connect to server: {t}", .{err})),
9971001 .response = undefined,
998 .buffer = reader_buffer,
1002 .transfer_buffer = reader_buffer,
1003 .decompress_buffer = &.{},
1004 .decompress = undefined,
9991005 } };
10001006 const request = &resource.http_request.request;
10011007 errdefer request.deinit();
......@@ -1019,6 +1025,7 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
10191025 .{ response.head.status, response.head.status.phrase() orelse "" },
10201026 ));
10211027
1028 resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
10221029 return;
10231030 }
10241031
......@@ -1027,13 +1034,12 @@ fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u
10271034 {
10281035 var transport_uri = uri;
10291036 transport_uri.scheme = uri.scheme["git+".len..];
1030 var session = git.Session.init(gpa, http_client, transport_uri, reader_buffer) catch |err| {
1031 return f.fail(f.location_tok, try eb.printString(
1032 "unable to discover remote git server capabilities: {s}",
1033 .{@errorName(err)},
1034 ));
1037 var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| {
1038 return f.fail(
1039 f.location_tok,
1040 try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
1041 );
10351042 };
1036 errdefer session.deinit();
10371043
10381044 const want_oid = want_oid: {
10391045 const want_ref =
src/Package/Fetch/git.zig+38-55
......@@ -644,7 +644,7 @@ pub const Session = struct {
644644 supports_agent: bool,
645645 supports_shallow: bool,
646646 object_format: Oid.Format,
647 allocator: Allocator,
647 arena: Allocator,
648648
649649 const agent = "zig/" ++ @import("builtin").zig_version_string;
650650 const agent_capability = std.fmt.comptimePrint("agent={s}\n", .{agent});
......@@ -652,7 +652,7 @@ pub const Session = struct {
652652 /// Initializes a client session and discovers the capabilities of the
653653 /// server for optimal transport.
654654 pub fn init(
655 allocator: Allocator,
655 arena: Allocator,
656656 transport: *std.http.Client,
657657 uri: std.Uri,
658658 /// Asserted to be at least `Packet.max_data_length`
......@@ -661,13 +661,12 @@ pub const Session = struct {
661661 assert(response_buffer.len >= Packet.max_data_length);
662662 var session: Session = .{
663663 .transport = transport,
664 .location = try .init(allocator, uri),
664 .location = try .init(arena, uri),
665665 .supports_agent = false,
666666 .supports_shallow = false,
667667 .object_format = .sha1,
668 .allocator = allocator,
668 .arena = arena,
669669 };
670 errdefer session.deinit();
671670 var capability_iterator: CapabilityIterator = undefined;
672671 try session.getCapabilities(&capability_iterator, response_buffer);
673672 defer capability_iterator.deinit();
......@@ -690,34 +689,24 @@ pub const Session = struct {
690689 return session;
691690 }
692691
693 pub fn deinit(session: *Session) void {
694 session.location.deinit(session.allocator);
695 session.* = undefined;
696 }
697
698692 /// An owned `std.Uri` representing the location of the server (base URI).
699693 const Location = struct {
700694 uri: std.Uri,
701695
702 fn init(allocator: Allocator, uri: std.Uri) !Location {
703 const scheme = try allocator.dupe(u8, uri.scheme);
704 errdefer allocator.free(scheme);
705 const user = if (uri.user) |user| try std.fmt.allocPrint(allocator, "{f}", .{
696 fn init(arena: Allocator, uri: std.Uri) !Location {
697 const scheme = try arena.dupe(u8, uri.scheme);
698 const user = if (uri.user) |user| try std.fmt.allocPrint(arena, "{f}", .{
706699 std.fmt.alt(user, .formatUser),
707700 }) else null;
708 errdefer if (user) |s| allocator.free(s);
709 const password = if (uri.password) |password| try std.fmt.allocPrint(allocator, "{f}", .{
701 const password = if (uri.password) |password| try std.fmt.allocPrint(arena, "{f}", .{
710702 std.fmt.alt(password, .formatPassword),
711703 }) else null;
712 errdefer if (password) |s| allocator.free(s);
713 const host = if (uri.host) |host| try std.fmt.allocPrint(allocator, "{f}", .{
704 const host = if (uri.host) |host| try std.fmt.allocPrint(arena, "{f}", .{
714705 std.fmt.alt(host, .formatHost),
715706 }) else null;
716 errdefer if (host) |s| allocator.free(s);
717 const path = try std.fmt.allocPrint(allocator, "{f}", .{
707 const path = try std.fmt.allocPrint(arena, "{f}", .{
718708 std.fmt.alt(uri.path, .formatPath),
719709 });
720 errdefer allocator.free(path);
721710 // The query and fragment are not used as part of the base server URI.
722711 return .{
723712 .uri = .{
......@@ -730,14 +719,6 @@ pub const Session = struct {
730719 },
731720 };
732721 }
733
734 fn deinit(loc: *Location, allocator: Allocator) void {
735 allocator.free(loc.uri.scheme);
736 if (loc.uri.user) |user| allocator.free(user.percent_encoded);
737 if (loc.uri.password) |password| allocator.free(password.percent_encoded);
738 if (loc.uri.host) |host| allocator.free(host.percent_encoded);
739 allocator.free(loc.uri.path.percent_encoded);
740 }
741722 };
742723
743724 /// Returns an iterator over capabilities supported by the server.
......@@ -745,16 +726,17 @@ pub const Session = struct {
745726 /// The `session.location` is updated if the server returns a redirect, so
746727 /// that subsequent session functions do not need to handle redirects.
747728 fn getCapabilities(session: *Session, it: *CapabilityIterator, response_buffer: []u8) !void {
729 const arena = session.arena;
748730 assert(response_buffer.len >= Packet.max_data_length);
749731 var info_refs_uri = session.location.uri;
750732 {
751 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{
733 const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
752734 std.fmt.alt(session.location.uri.path, .formatPath),
753735 });
754 defer session.allocator.free(session_uri_path);
755 info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "info/refs" }) };
736 info_refs_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{
737 "/", session_uri_path, "info/refs",
738 }) };
756739 }
757 defer session.allocator.free(info_refs_uri.path.percent_encoded);
758740 info_refs_uri.query = .{ .percent_encoded = "service=git-upload-pack" };
759741 info_refs_uri.fragment = null;
760742
......@@ -767,6 +749,7 @@ pub const Session = struct {
767749 },
768750 }),
769751 .reader = undefined,
752 .decompress = undefined,
770753 };
771754 errdefer it.deinit();
772755 const request = &it.request;
......@@ -777,19 +760,17 @@ pub const Session = struct {
777760 if (response.head.status != .ok) return error.ProtocolError;
778761 const any_redirects_occurred = request.redirect_behavior.remaining() < max_redirects;
779762 if (any_redirects_occurred) {
780 const request_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{
763 const request_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
781764 std.fmt.alt(request.uri.path, .formatPath),
782765 });
783 defer session.allocator.free(request_uri_path);
784766 if (!mem.endsWith(u8, request_uri_path, "/info/refs")) return error.UnparseableRedirect;
785767 var new_uri = request.uri;
786768 new_uri.path = .{ .percent_encoded = request_uri_path[0 .. request_uri_path.len - "/info/refs".len] };
787 const new_location: Location = try .init(session.allocator, new_uri);
788 session.location.deinit(session.allocator);
789 session.location = new_location;
769 session.location = try .init(arena, new_uri);
790770 }
791771
792 it.reader = response.reader(response_buffer);
772 const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
773 it.reader = response.readerDecompressing(response_buffer, &it.decompress, decompress_buffer);
793774 var state: enum { response_start, response_content } = .response_start;
794775 while (true) {
795776 // Some Git servers (at least GitHub) include an additional
......@@ -821,6 +802,7 @@ pub const Session = struct {
821802 const CapabilityIterator = struct {
822803 request: std.http.Client.Request,
823804 reader: *std.Io.Reader,
805 decompress: std.http.Decompress,
824806
825807 const Capability = struct {
826808 key: []const u8,
......@@ -864,16 +846,15 @@ pub const Session = struct {
864846
865847 /// Returns an iterator over refs known to the server.
866848 pub fn listRefs(session: Session, it: *RefIterator, options: ListRefsOptions) !void {
849 const arena = session.arena;
867850 assert(options.buffer.len >= Packet.max_data_length);
868851 var upload_pack_uri = session.location.uri;
869852 {
870 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{
853 const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
871854 std.fmt.alt(session.location.uri.path, .formatPath),
872855 });
873 defer session.allocator.free(session_uri_path);
874 upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) };
856 upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
875857 }
876 defer session.allocator.free(upload_pack_uri.path.percent_encoded);
877858 upload_pack_uri.query = null;
878859 upload_pack_uri.fragment = null;
879860
......@@ -883,16 +864,14 @@ pub const Session = struct {
883864 try Packet.write(.{ .data = agent_capability }, &body);
884865 }
885866 {
886 const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={t}\n", .{
867 const object_format_packet = try std.fmt.allocPrint(arena, "object-format={t}\n", .{
887868 session.object_format,
888869 });
889 defer session.allocator.free(object_format_packet);
890870 try Packet.write(.{ .data = object_format_packet }, &body);
891871 }
892872 try Packet.write(.delimiter, &body);
893873 for (options.ref_prefixes) |ref_prefix| {
894 const ref_prefix_packet = try std.fmt.allocPrint(session.allocator, "ref-prefix {s}\n", .{ref_prefix});
895 defer session.allocator.free(ref_prefix_packet);
874 const ref_prefix_packet = try std.fmt.allocPrint(arena, "ref-prefix {s}\n", .{ref_prefix});
896875 try Packet.write(.{ .data = ref_prefix_packet }, &body);
897876 }
898877 if (options.include_symrefs) {
......@@ -913,6 +892,7 @@ pub const Session = struct {
913892 }),
914893 .reader = undefined,
915894 .format = session.object_format,
895 .decompress = undefined,
916896 };
917897 const request = &it.request;
918898 errdefer request.deinit();
......@@ -920,13 +900,15 @@ pub const Session = struct {
920900
921901 var response = try request.receiveHead(options.buffer);
922902 if (response.head.status != .ok) return error.ProtocolError;
923 it.reader = response.reader(options.buffer);
903 const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
904 it.reader = response.readerDecompressing(options.buffer, &it.decompress, decompress_buffer);
924905 }
925906
926907 pub const RefIterator = struct {
927908 format: Oid.Format,
928909 request: std.http.Client.Request,
929910 reader: *std.Io.Reader,
911 decompress: std.http.Decompress,
930912
931913 pub const Ref = struct {
932914 oid: Oid,
......@@ -981,16 +963,15 @@ pub const Session = struct {
981963 /// Asserted to be at least `Packet.max_data_length`.
982964 response_buffer: []u8,
983965 ) !void {
966 const arena = session.arena;
984967 assert(response_buffer.len >= Packet.max_data_length);
985968 var upload_pack_uri = session.location.uri;
986969 {
987 const session_uri_path = try std.fmt.allocPrint(session.allocator, "{f}", .{
970 const session_uri_path = try std.fmt.allocPrint(arena, "{f}", .{
988971 std.fmt.alt(session.location.uri.path, .formatPath),
989972 });
990 defer session.allocator.free(session_uri_path);
991 upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(session.allocator, &.{ "/", session_uri_path, "git-upload-pack" }) };
973 upload_pack_uri.path = .{ .percent_encoded = try std.fs.path.resolvePosix(arena, &.{ "/", session_uri_path, "git-upload-pack" }) };
992974 }
993 defer session.allocator.free(upload_pack_uri.path.percent_encoded);
994975 upload_pack_uri.query = null;
995976 upload_pack_uri.fragment = null;
996977
......@@ -1000,8 +981,7 @@ pub const Session = struct {
1000981 try Packet.write(.{ .data = agent_capability }, &body);
1001982 }
1002983 {
1003 const object_format_packet = try std.fmt.allocPrint(session.allocator, "object-format={s}\n", .{@tagName(session.object_format)});
1004 defer session.allocator.free(object_format_packet);
984 const object_format_packet = try std.fmt.allocPrint(arena, "object-format={s}\n", .{@tagName(session.object_format)});
1005985 try Packet.write(.{ .data = object_format_packet }, &body);
1006986 }
1007987 try Packet.write(.delimiter, &body);
......@@ -1031,6 +1011,7 @@ pub const Session = struct {
10311011 .input = undefined,
10321012 .reader = undefined,
10331013 .remaining_len = undefined,
1014 .decompress = undefined,
10341015 };
10351016 const request = &fs.request;
10361017 errdefer request.deinit();
......@@ -1040,7 +1021,8 @@ pub const Session = struct {
10401021 var response = try request.receiveHead(&.{});
10411022 if (response.head.status != .ok) return error.ProtocolError;
10421023
1043 const reader = response.reader(response_buffer);
1024 const decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
1025 const reader = response.readerDecompressing(response_buffer, &fs.decompress, decompress_buffer);
10441026 // We are not interested in any of the sections of the returned fetch
10451027 // data other than the packfile section, since we aren't doing anything
10461028 // complex like ref negotiation (this is a fresh clone).
......@@ -1079,6 +1061,7 @@ pub const Session = struct {
10791061 reader: std.Io.Reader,
10801062 err: ?Error = null,
10811063 remaining_len: usize,
1064 decompress: std.http.Decompress,
10821065
10831066 pub fn deinit(fs: *FetchStream) void {
10841067 fs.request.deinit();