authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-28 22:12:46-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-31 22:10:11-07:00
log05ce1f99a60df0bcede00af1c98a1e5fdce35051
treeb9dfb4f32f94125bda0a1ad925fec8a66179f9ed
parent2569f4ff8514f47730c1abd42691f8fc1cdd22d5

compiler: update to new flate API

7 files changed, 122 insertions(+), 107 deletions(-)

lib/std/Io/Reader.zig+9
...@@ -1709,6 +1709,15 @@ fn failingDiscard(r: *Reader, limit: Limit) Error!usize {...@@ -1709,6 +1709,15 @@ fn failingDiscard(r: *Reader, limit: Limit) Error!usize {
1709 return error.ReadFailed;1709 return error.ReadFailed;
1710}1710}
17111711
1712pub fn adaptToOldInterface(r: *Reader) std.Io.AnyReader {
1713 return .{ .context = r, .readFn = derpRead };
1714}
1715
1716fn derpRead(context: *const anyopaque, buffer: []u8) anyerror!usize {
1717 const r: *Reader = @constCast(@alignCast(@ptrCast(context)));
1718 return r.readSliceShort(buffer);
1719}
1720
1712test "readAlloc when the backing reader provides one byte at a time" {1721test "readAlloc when the backing reader provides one byte at a time" {
1713 const str = "This is a test";1722 const str = "This is a test";
1714 var tiny_buffer: [1]u8 = undefined;1723 var tiny_buffer: [1]u8 = undefined;
lib/std/compress/flate/Decompress.zig+5-5
...@@ -23,7 +23,7 @@ dst_dec: DistanceDecoder,...@@ -23,7 +23,7 @@ dst_dec: DistanceDecoder,
23final_block: bool,23final_block: bool,
24state: State,24state: State,
2525
26read_err: ?Error,26err: ?Error,
2727
28const BlockType = enum(u2) {28const BlockType = enum(u2) {
29 stored = 0,29 stored = 0,
...@@ -74,7 +74,7 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress {...@@ -74,7 +74,7 @@ pub fn init(input: *Reader, container: Container, buffer: []u8) Decompress {
74 .dst_dec = .{},74 .dst_dec = .{},
75 .final_block = false,75 .final_block = false,
76 .state = .protocol_header,76 .state = .protocol_header,
77 .read_err = null,77 .err = null,
78 };78 };
79}79}
8080
...@@ -153,7 +153,7 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!us...@@ -153,7 +153,7 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!us
153 if (d.state == .end) {153 if (d.state == .end) {
154 return error.EndOfStream;154 return error.EndOfStream;
155 } else {155 } else {
156 d.read_err = error.EndOfStream;156 d.err = error.EndOfStream;
157 return error.ReadFailed;157 return error.ReadFailed;
158 }158 }
159 },159 },
...@@ -161,7 +161,7 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!us...@@ -161,7 +161,7 @@ pub fn stream(r: *Reader, w: *Writer, limit: std.Io.Limit) Reader.StreamError!us
161 else => |e| {161 else => |e| {
162 // In the event of an error, state is unmodified so that it can be162 // In the event of an error, state is unmodified so that it can be
163 // better used to diagnose the failure.163 // better used to diagnose the failure.
164 d.read_err = e;164 d.err = e;
165 return error.ReadFailed;165 return error.ReadFailed;
166 },166 },
167 };167 };
...@@ -1179,7 +1179,7 @@ fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !vo...@@ -1179,7 +1179,7 @@ fn testFailure(container: Container, in: []const u8, expected_err: anyerror) !vo
11791179
1180 var decompress: Decompress = .init(&reader, container, &.{});1180 var decompress: Decompress = .init(&reader, container, &.{});
1181 try testing.expectError(error.ReadFailed, decompress.reader.streamRemaining(&aw.writer));1181 try testing.expectError(error.ReadFailed, decompress.reader.streamRemaining(&aw.writer));
1182 try testing.expectEqual(expected_err, decompress.read_err orelse return error.TestFailed);1182 try testing.expectEqual(expected_err, decompress.err orelse return error.TestFailed);
1183}1183}
11841184
1185fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {1185fn testDecompress(container: Container, compressed: []const u8, expected_plain: []const u8) !void {
lib/std/http/Client.zig+9-10
...@@ -405,8 +405,8 @@ pub const RequestTransfer = union(enum) {...@@ -405,8 +405,8 @@ pub const RequestTransfer = union(enum) {
405405
406/// The decompressor for response messages.406/// The decompressor for response messages.
407pub const Compression = union(enum) {407pub const Compression = union(enum) {
408 deflate: std.compress.flate.Decompress,408 //deflate: std.compress.flate.Decompress,
409 gzip: std.compress.flate.Decompress,409 //gzip: std.compress.flate.Decompress,
410 // https://github.com/ziglang/zig/issues/18937410 // https://github.com/ziglang/zig/issues/18937
411 //zstd: ZstdDecompressor,411 //zstd: ZstdDecompressor,
412 none: void,412 none: void,
...@@ -1074,12 +1074,10 @@ pub const Request = struct {...@@ -1074,12 +1074,10 @@ pub const Request = struct {
1074 switch (req.response.transfer_compression) {1074 switch (req.response.transfer_compression) {
1075 .identity => req.response.compression = .none,1075 .identity => req.response.compression = .none,
1076 .compress, .@"x-compress" => return error.CompressionUnsupported,1076 .compress, .@"x-compress" => return error.CompressionUnsupported,
1077 .deflate => req.response.compression = .{1077 // I'm about to upstream my http.Client rewrite
1078 .deflate = std.compress.zlib.decompressor(req.transferReader()),1078 .deflate => return error.CompressionUnsupported,
1079 },1079 // I'm about to upstream my http.Client rewrite
1080 .gzip, .@"x-gzip" => req.response.compression = .{1080 .gzip, .@"x-gzip" => return error.CompressionUnsupported,
1081 .gzip = std.compress.gzip.decompressor(req.transferReader()),
1082 },
1083 // https://github.com/ziglang/zig/issues/189371081 // https://github.com/ziglang/zig/issues/18937
1084 //.zstd => req.response.compression = .{1082 //.zstd => req.response.compression = .{
1085 // .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),1083 // .zstd = std.compress.zstd.decompressStream(req.client.allocator, req.transferReader()),
...@@ -1105,8 +1103,9 @@ pub const Request = struct {...@@ -1105,8 +1103,9 @@ pub const Request = struct {
1105 /// Reads data from the response body. Must be called after `wait`.1103 /// Reads data from the response body. Must be called after `wait`.
1106 pub fn read(req: *Request, buffer: []u8) ReadError!usize {1104 pub fn read(req: *Request, buffer: []u8) ReadError!usize {
1107 const out_index = switch (req.response.compression) {1105 const out_index = switch (req.response.compression) {
1108 .deflate => |*deflate| deflate.read(buffer) catch return error.DecompressionFailure,1106 // I'm about to upstream my http client rewrite
1109 .gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure,1107 //.deflate => |*deflate| deflate.readSlice(buffer) catch return error.DecompressionFailure,
1108 //.gzip => |*gzip| gzip.read(buffer) catch return error.DecompressionFailure,
1110 // https://github.com/ziglang/zig/issues/189371109 // https://github.com/ziglang/zig/issues/18937
1111 //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,1110 //.zstd => |*zstd| zstd.read(buffer) catch return error.DecompressionFailure,
1112 else => try req.transferRead(buffer),1111 else => try req.transferRead(buffer),
lib/std/zip.zig+45-39
...@@ -7,8 +7,9 @@ const builtin = @import("builtin");...@@ -7,8 +7,9 @@ const builtin = @import("builtin");
7const std = @import("std");7const std = @import("std");
8const File = std.fs.File;8const File = std.fs.File;
9const is_le = builtin.target.cpu.arch.endian() == .little;9const is_le = builtin.target.cpu.arch.endian() == .little;
10const Writer = std.io.Writer;10const Writer = std.Io.Writer;
11const Reader = std.io.Reader;11const Reader = std.Io.Reader;
12const flate = std.compress.flate;
1213
13pub const CompressionMethod = enum(u16) {14pub const CompressionMethod = enum(u16) {
14 store = 0,15 store = 0,
...@@ -117,6 +118,7 @@ pub const EndRecord = extern struct {...@@ -117,6 +118,7 @@ pub const EndRecord = extern struct {
117 pub const FindFileError = File.GetEndPosError || File.SeekError || File.ReadError || error{118 pub const FindFileError = File.GetEndPosError || File.SeekError || File.ReadError || error{
118 ZipNoEndRecord,119 ZipNoEndRecord,
119 EndOfStream,120 EndOfStream,
121 ReadFailed,
120 };122 };
121123
122 pub fn findFile(fr: *File.Reader) FindFileError!EndRecord {124 pub fn findFile(fr: *File.Reader) FindFileError!EndRecord {
...@@ -137,8 +139,7 @@ pub const EndRecord = extern struct {...@@ -137,8 +139,7 @@ pub const EndRecord = extern struct {
137139
138 try fr.seekTo(end_pos - @as(u64, new_loaded_len));140 try fr.seekTo(end_pos - @as(u64, new_loaded_len));
139 const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];141 const read_buf: []u8 = buf[buf.len - new_loaded_len ..][0..read_len];
140 var br = fr.interface().unbuffered();142 fr.interface.readSliceAll(read_buf) catch |err| switch (err) {
141 br.readSlice(read_buf) catch |err| switch (err) {
142 error.ReadFailed => return fr.err.?,143 error.ReadFailed => return fr.err.?,
143 error.EndOfStream => return error.EndOfStream,144 error.EndOfStream => return error.EndOfStream,
144 };145 };
...@@ -164,7 +165,7 @@ pub const EndRecord = extern struct {...@@ -164,7 +165,7 @@ pub const EndRecord = extern struct {
164pub const Decompress = struct {165pub const Decompress = struct {
165 interface: Reader,166 interface: Reader,
166 state: union {167 state: union {
167 inflate: std.compress.flate.Decompress,168 inflate: flate.Decompress,
168 store: *Reader,169 store: *Reader,
169 },170 },
170171
...@@ -201,7 +202,7 @@ pub const Decompress = struct {...@@ -201,7 +202,7 @@ pub const Decompress = struct {
201202
202 fn streamDeflate(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize {203 fn streamDeflate(r: *Reader, w: *Writer, limit: std.io.Limit) Reader.StreamError!usize {
203 const d: *Decompress = @fieldParentPtr("interface", r);204 const d: *Decompress = @fieldParentPtr("interface", r);
204 return std.compress.flate.Decompress.read(&d.inflate, w, limit);205 return flate.Decompress.read(&d.inflate, w, limit);
205 }206 }
206};207};
207208
...@@ -305,7 +306,7 @@ pub const Iterator = struct {...@@ -305,7 +306,7 @@ pub const Iterator = struct {
305 if (locator_end_offset > stream_len)306 if (locator_end_offset > stream_len)
306 return error.ZipTruncated;307 return error.ZipTruncated;
307 try input.seekTo(stream_len - locator_end_offset);308 try input.seekTo(stream_len - locator_end_offset);
308 const locator = input.interface.takeStructEndian(EndLocator64, .little) catch |err| switch (err) {309 const locator = input.interface.takeStruct(EndLocator64, .little) catch |err| switch (err) {
309 error.ReadFailed => return input.err.?,310 error.ReadFailed => return input.err.?,
310 error.EndOfStream => return error.EndOfStream,311 error.EndOfStream => return error.EndOfStream,
311 };312 };
...@@ -318,7 +319,7 @@ pub const Iterator = struct {...@@ -318,7 +319,7 @@ pub const Iterator = struct {
318319
319 try input.seekTo(locator.record_file_offset);320 try input.seekTo(locator.record_file_offset);
320321
321 const record64 = input.interface.takeStructEndian(EndRecord64, .little) catch |err| switch (err) {322 const record64 = input.interface.takeStruct(EndRecord64, .little) catch |err| switch (err) {
322 error.ReadFailed => return input.err.?,323 error.ReadFailed => return input.err.?,
323 error.EndOfStream => return error.EndOfStream,324 error.EndOfStream => return error.EndOfStream,
324 };325 };
...@@ -374,7 +375,7 @@ pub const Iterator = struct {...@@ -374,7 +375,7 @@ pub const Iterator = struct {
374 const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;375 const header_zip_offset = self.cd_zip_offset + self.cd_record_offset;
375 const input = self.input;376 const input = self.input;
376 try input.seekTo(header_zip_offset);377 try input.seekTo(header_zip_offset);
377 const header = input.interface.takeStructEndian(CentralDirectoryFileHeader, .little) catch |err| switch (err) {378 const header = input.interface.takeStruct(CentralDirectoryFileHeader, .little) catch |err| switch (err) {
378 error.ReadFailed => return input.err.?,379 error.ReadFailed => return input.err.?,
379 error.EndOfStream => return error.EndOfStream,380 error.EndOfStream => return error.EndOfStream,
380 };381 };
...@@ -405,7 +406,7 @@ pub const Iterator = struct {...@@ -405,7 +406,7 @@ pub const Iterator = struct {
405 const extra = extra_buf[0..header.extra_len];406 const extra = extra_buf[0..header.extra_len];
406407
407 try input.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);408 try input.seekTo(header_zip_offset + @sizeOf(CentralDirectoryFileHeader) + header.filename_len);
408 input.interface.readSlice(extra) catch |err| switch (err) {409 input.interface.readSliceAll(extra) catch |err| switch (err) {
409 error.ReadFailed => return input.err.?,410 error.ReadFailed => return input.err.?,
410 error.EndOfStream => return error.EndOfStream,411 error.EndOfStream => return error.EndOfStream,
411 };412 };
...@@ -460,7 +461,7 @@ pub const Iterator = struct {...@@ -460,7 +461,7 @@ pub const Iterator = struct {
460 options: ExtractOptions,461 options: ExtractOptions,
461 filename_buf: []u8,462 filename_buf: []u8,
462 dest: std.fs.Dir,463 dest: std.fs.Dir,
463 ) !u32 {464 ) !void {
464 if (filename_buf.len < self.filename_len)465 if (filename_buf.len < self.filename_len)
465 return error.ZipInsufficientBuffer;466 return error.ZipInsufficientBuffer;
466 switch (self.compression_method) {467 switch (self.compression_method) {
...@@ -470,13 +471,13 @@ pub const Iterator = struct {...@@ -470,13 +471,13 @@ pub const Iterator = struct {
470 const filename = filename_buf[0..self.filename_len];471 const filename = filename_buf[0..self.filename_len];
471 {472 {
472 try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));473 try stream.seekTo(self.header_zip_offset + @sizeOf(CentralDirectoryFileHeader));
473 try stream.interface.readSlice(filename);474 try stream.interface.readSliceAll(filename);
474 }475 }
475476
476 const local_data_header_offset: u64 = local_data_header_offset: {477 const local_data_header_offset: u64 = local_data_header_offset: {
477 const local_header = blk: {478 const local_header = blk: {
478 try stream.seekTo(self.file_offset);479 try stream.seekTo(self.file_offset);
479 break :blk try stream.interface.takeStructEndian(LocalFileHeader, .little);480 break :blk try stream.interface.takeStruct(LocalFileHeader, .little);
480 };481 };
481 if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))482 if (!std.mem.eql(u8, &local_header.signature, &local_file_header_sig))
482 return error.ZipBadFileOffset;483 return error.ZipBadFileOffset;
...@@ -502,7 +503,7 @@ pub const Iterator = struct {...@@ -502,7 +503,7 @@ pub const Iterator = struct {
502503
503 {504 {
504 try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);505 try stream.seekTo(self.file_offset + @sizeOf(LocalFileHeader) + local_header.filename_len);
505 try stream.interface.readSlice(extra);506 try stream.interface.readSliceAll(extra);
506 }507 }
507508
508 var extra_offset: usize = 0;509 var extra_offset: usize = 0;
...@@ -550,7 +551,7 @@ pub const Iterator = struct {...@@ -550,7 +551,7 @@ pub const Iterator = struct {
550 if (self.uncompressed_size != 0)551 if (self.uncompressed_size != 0)
551 return error.ZipBadDirectorySize;552 return error.ZipBadDirectorySize;
552 try dest.makePath(filename[0 .. filename.len - 1]);553 try dest.makePath(filename[0 .. filename.len - 1]);
553 return std.hash.Crc32.hash(&.{});554 return;
554 }555 }
555556
556 const out_file = blk: {557 const out_file = blk: {
...@@ -564,31 +565,36 @@ pub const Iterator = struct {...@@ -564,31 +565,36 @@ pub const Iterator = struct {
564 break :blk try dest.createFile(filename, .{ .exclusive = true });565 break :blk try dest.createFile(filename, .{ .exclusive = true });
565 };566 };
566 defer out_file.close();567 defer out_file.close();
567 var file_writer = out_file.writer();568 var out_file_buffer: [1024]u8 = undefined;
568 var file_bw = file_writer.writer(&.{});569 var file_writer = out_file.writer(&out_file_buffer);
569 const local_data_file_offset: u64 =570 const local_data_file_offset: u64 =
570 @as(u64, self.file_offset) +571 @as(u64, self.file_offset) +
571 @as(u64, @sizeOf(LocalFileHeader)) +572 @as(u64, @sizeOf(LocalFileHeader)) +
572 local_data_header_offset;573 local_data_header_offset;
573 try stream.seekTo(local_data_file_offset);574 try stream.seekTo(local_data_file_offset);
574 var limited_file_reader = stream.interface.limited(.limited(self.compressed_size));575
575 var file_read_buffer: [1000]u8 = undefined;576 // TODO limit based on self.compressed_size
576 var decompress_read_buffer: [1000]u8 = undefined;577
577 var limited_br = limited_file_reader.reader().buffered(&file_read_buffer);578 switch (self.compression_method) {
578 var decompress: Decompress = undefined;579 .store => {
579 var decompress_br = decompress.readable(&limited_br, self.compression_method, &decompress_read_buffer);580 stream.interface.streamExact(&file_writer.interface, self.uncompressed_size) catch |err| switch (err) {
580 const start_out = file_bw.count;581 error.ReadFailed => return stream.err.?,
581 var hash_writer = file_bw.hashed(std.hash.Crc32.init());582 error.WriteFailed => return file_writer.err.?,
582 var hash_bw = hash_writer.writer(&.{});583 error.EndOfStream => return error.ZipDecompressTruncated,
583 decompress_br.readAll(&hash_bw, .limited(self.uncompressed_size)) catch |err| switch (err) {584 };
584 error.ReadFailed => return stream.err.?,585 },
585 error.WriteFailed => return file_writer.err.?,586 .deflate => {
586 error.EndOfStream => return error.ZipDecompressTruncated,587 var flate_buffer: [flate.max_window_len]u8 = undefined;
587 };588 var decompress: flate.Decompress = .init(&stream.interface, .raw, &flate_buffer);
588 if (limited_file_reader.remaining.nonzero()) return error.ZipDecompressTruncated;589 decompress.reader.streamExact(&file_writer.interface, self.uncompressed_size) catch |err| switch (err) {
589 const written = file_bw.count - start_out;590 error.ReadFailed => return stream.err.?,
590 if (written != self.uncompressed_size) return error.ZipUncompressSizeMismatch;591 error.WriteFailed => return file_writer.err orelse decompress.err.?,
591 return hash_writer.hasher.final();592 error.EndOfStream => return error.ZipDecompressTruncated,
593 };
594 },
595 else => return error.UnsupportedCompressionMethod,
596 }
597 try file_writer.end();
592 }598 }
593 };599 };
594};600};
...@@ -636,19 +642,19 @@ pub const ExtractOptions = struct {...@@ -636,19 +642,19 @@ pub const ExtractOptions = struct {
636 /// Allow filenames within the zip to use backslashes. Back slashes are normalized642 /// Allow filenames within the zip to use backslashes. Back slashes are normalized
637 /// to forward slashes before forwarding them to platform APIs.643 /// to forward slashes before forwarding them to platform APIs.
638 allow_backslashes: bool = false,644 allow_backslashes: bool = false,
639
640 diagnostics: ?*Diagnostics = null,645 diagnostics: ?*Diagnostics = null,
646 verify_checksums: bool = false,
641};647};
642648
643/// Extract the zipped files to the given `dest` directory.649/// Extract the zipped files to the given `dest` directory.
644pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !void {650pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !void {
651 if (options.verify_checksums) @panic("TODO unimplemented");
652
645 var iter = try Iterator.init(fr);653 var iter = try Iterator.init(fr);
646654
647 var filename_buf: [std.fs.max_path_bytes]u8 = undefined;655 var filename_buf: [std.fs.max_path_bytes]u8 = undefined;
648 while (try iter.next()) |entry| {656 while (try iter.next()) |entry| {
649 const crc32 = try entry.extract(fr, options, &filename_buf, dest);657 try entry.extract(fr, options, &filename_buf, dest);
650 if (crc32 != entry.crc32)
651 return error.ZipCrcMismatch;
652 if (options.diagnostics) |d| {658 if (options.diagnostics) |d| {
653 try d.nextFilename(filename_buf[0..entry.filename_len]);659 try d.nextFilename(filename_buf[0..entry.filename_len]);
654 }660 }
src/Package/Fetch.zig+15-10
...@@ -1203,12 +1203,11 @@ fn unpackResource(...@@ -1203,12 +1203,11 @@ fn unpackResource(
1203 return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);1203 return unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1204 },1204 },
1205 .@"tar.gz" => {1205 .@"tar.gz" => {
1206 const reader = resource.reader();1206 var adapter_buffer: [std.crypto.tls.max_ciphertext_record_len]u8 = undefined;
1207 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);1207 var adapter = resource.reader().adaptToNewApi(&adapter_buffer);
1208 var dcp = std.compress.gzip.decompressor(br.reader());1208 var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1209 var adapter_buffer: [1024]u8 = undefined;1209 var decompress: std.compress.flate.Decompress = .init(&adapter.new_interface, .gzip, &flate_buffer);
1210 var adapter = dcp.reader().adaptToNewApi(&adapter_buffer);1210 return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1211 return try unpackTarball(f, tmp_directory.handle, &adapter.new_interface);
1212 },1211 },
1213 .@"tar.xz" => {1212 .@"tar.xz" => {
1214 const gpa = f.arena.child_allocator;1213 const gpa = f.arena.child_allocator;
...@@ -1352,7 +1351,10 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {...@@ -1352,7 +1351,10 @@ fn unzip(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!UnpackResult {
1352 ));1351 ));
1353 defer zip_file.close();1352 defer zip_file.close();
13541353
1355 std.zip.extract(out_dir, zip_file.seekableStream(), .{1354 var zip_file_buffer: [1024]u8 = undefined;
1355 var zip_file_reader = zip_file.reader(&zip_file_buffer);
1356
1357 std.zip.extract(out_dir, &zip_file_reader, .{
1356 .allow_backslashes = true,1358 .allow_backslashes = true,
1357 .diagnostics = &diagnostics,1359 .diagnostics = &diagnostics,
1358 }) catch |err| return f.fail(f.location_tok, try eb.printString(1360 }) catch |err| return f.fail(f.location_tok, try eb.printString(
...@@ -1384,23 +1386,26 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!U...@@ -1384,23 +1386,26 @@ fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource.Git) anyerror!U
1384 defer pack_dir.close();1386 defer pack_dir.close();
1385 var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });1387 var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
1386 defer pack_file.close();1388 defer pack_file.close();
1387 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();1389 var pack_file_buffer: [4096]u8 = undefined;
1390 var fifo = std.fifo.LinearFifo(u8, .{ .Slice = {} }).init(&pack_file_buffer);
1388 try fifo.pump(resource.fetch_stream.reader(), pack_file.deprecatedWriter());1391 try fifo.pump(resource.fetch_stream.reader(), pack_file.deprecatedWriter());
13891392
1393 var pack_file_reader = pack_file.reader(&pack_file_buffer);
1394
1390 var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });1395 var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
1391 defer index_file.close();1396 defer index_file.close();
1392 {1397 {
1393 const index_prog_node = f.prog_node.start("Index pack", 0);1398 const index_prog_node = f.prog_node.start("Index pack", 0);
1394 defer index_prog_node.end();1399 defer index_prog_node.end();
1395 var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());1400 var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());
1396 try git.indexPack(gpa, object_format, pack_file, index_buffered_writer.writer());1401 try git.indexPack(gpa, object_format, &pack_file_reader, index_buffered_writer.writer());
1397 try index_buffered_writer.flush();1402 try index_buffered_writer.flush();
1398 }1403 }
13991404
1400 {1405 {
1401 const checkout_prog_node = f.prog_node.start("Checkout", 0);1406 const checkout_prog_node = f.prog_node.start("Checkout", 0);
1402 defer checkout_prog_node.end();1407 defer checkout_prog_node.end();
1403 var repository = try git.Repository.init(gpa, object_format, pack_file, index_file);1408 var repository = try git.Repository.init(gpa, object_format, &pack_file_reader, index_file);
1404 defer repository.deinit();1409 defer repository.deinit();
1405 var diagnostics: git.Diagnostics = .{ .allocator = arena };1410 var diagnostics: git.Diagnostics = .{ .allocator = arena };
1406 try repository.checkout(out_dir, resource.want_oid, &diagnostics);1411 try repository.checkout(out_dir, resource.want_oid, &diagnostics);
src/Package/Fetch/git.zig+32-35
...@@ -73,7 +73,7 @@ pub const Oid = union(Format) {...@@ -73,7 +73,7 @@ pub const Oid = union(Format) {
73 };73 };
74 }74 }
7575
76 pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid {76 pub fn readBytes(oid_format: Format, reader: anytype) !Oid {
77 return switch (oid_format) {77 return switch (oid_format) {
78 inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())),78 inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())),
79 };79 };
...@@ -166,7 +166,7 @@ pub const Diagnostics = struct {...@@ -166,7 +166,7 @@ pub const Diagnostics = struct {
166pub const Repository = struct {166pub const Repository = struct {
167 odb: Odb,167 odb: Odb,
168168
169 pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository {169 pub fn init(allocator: Allocator, format: Oid.Format, pack_file: *std.fs.File.Reader, index_file: std.fs.File) !Repository {
170 return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) };170 return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) };
171 }171 }
172172
...@@ -335,14 +335,14 @@ pub const Repository = struct {...@@ -335,14 +335,14 @@ pub const Repository = struct {
335/// [pack-format](https://git-scm.com/docs/pack-format).335/// [pack-format](https://git-scm.com/docs/pack-format).
336const Odb = struct {336const Odb = struct {
337 format: Oid.Format,337 format: Oid.Format,
338 pack_file: std.fs.File,338 pack_file: *std.fs.File.Reader,
339 index_header: IndexHeader,339 index_header: IndexHeader,
340 index_file: std.fs.File,340 index_file: std.fs.File,
341 cache: ObjectCache = .{},341 cache: ObjectCache = .{},
342 allocator: Allocator,342 allocator: Allocator,
343343
344 /// Initializes the database from open pack and index files.344 /// Initializes the database from open pack and index files.
345 fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb {345 fn init(allocator: Allocator, format: Oid.Format, pack_file: *std.fs.File.Reader, index_file: std.fs.File) !Odb {
346 try pack_file.seekTo(0);346 try pack_file.seekTo(0);
347 try index_file.seekTo(0);347 try index_file.seekTo(0);
348 const index_header = try IndexHeader.read(index_file.deprecatedReader());348 const index_header = try IndexHeader.read(index_file.deprecatedReader());
...@@ -362,14 +362,14 @@ const Odb = struct {...@@ -362,14 +362,14 @@ const Odb = struct {
362362
363 /// Reads the object at the current position in the database.363 /// Reads the object at the current position in the database.
364 fn readObject(odb: *Odb) !Object {364 fn readObject(odb: *Odb) !Object {
365 var base_offset = try odb.pack_file.getPos();365 var base_offset = odb.pack_file.logicalPos();
366 var base_header: EntryHeader = undefined;366 var base_header: EntryHeader = undefined;
367 var delta_offsets: std.ArrayListUnmanaged(u64) = .empty;367 var delta_offsets: std.ArrayListUnmanaged(u64) = .empty;
368 defer delta_offsets.deinit(odb.allocator);368 defer delta_offsets.deinit(odb.allocator);
369 const base_object = while (true) {369 const base_object = while (true) {
370 if (odb.cache.get(base_offset)) |base_object| break base_object;370 if (odb.cache.get(base_offset)) |base_object| break base_object;
371371
372 base_header = try EntryHeader.read(odb.format, odb.pack_file.deprecatedReader());372 base_header = try EntryHeader.read(odb.format, odb.pack_file.interface.adaptToOldInterface());
373 switch (base_header) {373 switch (base_header) {
374 .ofs_delta => |ofs_delta| {374 .ofs_delta => |ofs_delta| {
375 try delta_offsets.append(odb.allocator, base_offset);375 try delta_offsets.append(odb.allocator, base_offset);
...@@ -379,10 +379,10 @@ const Odb = struct {...@@ -379,10 +379,10 @@ const Odb = struct {
379 .ref_delta => |ref_delta| {379 .ref_delta => |ref_delta| {
380 try delta_offsets.append(odb.allocator, base_offset);380 try delta_offsets.append(odb.allocator, base_offset);
381 try odb.seekOid(ref_delta.base_object);381 try odb.seekOid(ref_delta.base_object);
382 base_offset = try odb.pack_file.getPos();382 base_offset = odb.pack_file.logicalPos();
383 },383 },
384 else => {384 else => {
385 const base_data = try readObjectRaw(odb.allocator, odb.pack_file.deprecatedReader(), base_header.uncompressedLength());385 const base_data = try readObjectRaw(odb.allocator, &odb.pack_file.interface, base_header.uncompressedLength());
386 errdefer odb.allocator.free(base_data);386 errdefer odb.allocator.free(base_data);
387 const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };387 const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
388 try odb.cache.put(odb.allocator, base_offset, base_object);388 try odb.cache.put(odb.allocator, base_offset, base_object);
...@@ -1227,7 +1227,7 @@ const IndexEntry = struct {...@@ -1227,7 +1227,7 @@ const IndexEntry = struct {
12271227
1228/// Writes out a version 2 index for the given packfile, as documented in1228/// Writes out a version 2 index for the given packfile, as documented in
1229/// [pack-format](https://git-scm.com/docs/pack-format).1229/// [pack-format](https://git-scm.com/docs/pack-format).
1230pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void {1230pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: *std.fs.File.Reader, index_writer: anytype) !void {
1231 try pack.seekTo(0);1231 try pack.seekTo(0);
12321232
1233 var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;1233 var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
...@@ -1324,12 +1324,11 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in...@@ -1324,12 +1324,11 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
1324fn indexPackFirstPass(1324fn indexPackFirstPass(
1325 allocator: Allocator,1325 allocator: Allocator,
1326 format: Oid.Format,1326 format: Oid.Format,
1327 pack: std.fs.File,1327 pack: *std.fs.File.Reader,
1328 index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),1328 index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
1329 pending_deltas: *std.ArrayListUnmanaged(IndexEntry),1329 pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
1330) !Oid {1330) !Oid {
1331 var pack_buffered_reader = std.io.bufferedReader(pack.deprecatedReader());1331 var pack_counting_reader = std.io.countingReader(pack.interface.adaptToOldInterface());
1332 var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
1333 var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));1332 var pack_hashed_reader = hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
1334 const pack_reader = pack_hashed_reader.reader();1333 const pack_reader = pack_hashed_reader.reader();
13351334
...@@ -1340,15 +1339,19 @@ fn indexPackFirstPass(...@@ -1340,15 +1339,19 @@ fn indexPackFirstPass(
1340 const entry_offset = pack_counting_reader.bytes_read;1339 const entry_offset = pack_counting_reader.bytes_read;
1341 var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init());1340 var entry_crc32_reader = hashedReader(pack_reader, std.hash.Crc32.init());
1342 const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());1341 const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());
1342 var adapter_buffer: [1024]u8 = undefined;
1343 var adapter = entry_crc32_reader.reader().adaptToNewApi(&adapter_buffer);
1344 var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1345 var entry_decompress_stream: std.compress.flate.Decompress = .init(&adapter.new_interface, .zlib, &flate_buffer);
1346 const old = entry_decompress_stream.reader.adaptToOldInterface();
1347 var entry_counting_reader = std.io.countingReader(old);
1343 switch (entry_header) {1348 switch (entry_header) {
1344 .commit, .tree, .blob, .tag => |object| {1349 .commit, .tree, .blob, .tag => |object| {
1345 var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1346 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1347 var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format));1350 var entry_hashed_writer = hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
1348 const entry_writer = entry_hashed_writer.writer();1351 const entry_writer = entry_hashed_writer.writer();
1349 // The object header is not included in the pack data but is1352 // The object header is not included in the pack data but is
1350 // part of the object's ID1353 // part of the object's ID
1351 try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length });1354 try entry_writer.print("{s} {d}\x00", .{ @tagName(entry_header), object.uncompressed_length });
1352 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();1355 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1353 try fifo.pump(entry_counting_reader.reader(), entry_writer);1356 try fifo.pump(entry_counting_reader.reader(), entry_writer);
1354 if (entry_counting_reader.bytes_read != object.uncompressed_length) {1357 if (entry_counting_reader.bytes_read != object.uncompressed_length) {
...@@ -1361,8 +1364,6 @@ fn indexPackFirstPass(...@@ -1361,8 +1364,6 @@ fn indexPackFirstPass(
1361 });1364 });
1362 },1365 },
1363 inline .ofs_delta, .ref_delta => |delta| {1366 inline .ofs_delta, .ref_delta => |delta| {
1364 var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
1365 var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
1366 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();1367 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
1367 try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);1368 try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
1368 if (entry_counting_reader.bytes_read != delta.uncompressed_length) {1369 if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
...@@ -1377,7 +1378,7 @@ fn indexPackFirstPass(...@@ -1377,7 +1378,7 @@ fn indexPackFirstPass(
1377 }1378 }
13781379
1379 const pack_checksum = pack_hashed_reader.hasher.finalResult();1380 const pack_checksum = pack_hashed_reader.hasher.finalResult();
1380 const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader());1381 const recorded_checksum = try Oid.readBytes(format, pack.interface.adaptToOldInterface());
1381 if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) {1382 if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) {
1382 return error.CorruptedPack;1383 return error.CorruptedPack;
1383 }1384 }
...@@ -1394,7 +1395,7 @@ fn indexPackFirstPass(...@@ -1394,7 +1395,7 @@ fn indexPackFirstPass(
1394fn indexPackHashDelta(1395fn indexPackHashDelta(
1395 allocator: Allocator,1396 allocator: Allocator,
1396 format: Oid.Format,1397 format: Oid.Format,
1397 pack: std.fs.File,1398 pack: *std.fs.File.Reader,
1398 delta: IndexEntry,1399 delta: IndexEntry,
1399 index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),1400 index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
1400 cache: *ObjectCache,1401 cache: *ObjectCache,
...@@ -1408,7 +1409,7 @@ fn indexPackHashDelta(...@@ -1408,7 +1409,7 @@ fn indexPackHashDelta(
1408 if (cache.get(base_offset)) |base_object| break base_object;1409 if (cache.get(base_offset)) |base_object| break base_object;
14091410
1410 try pack.seekTo(base_offset);1411 try pack.seekTo(base_offset);
1411 base_header = try EntryHeader.read(format, pack.deprecatedReader());1412 base_header = try EntryHeader.read(format, pack.interface.adaptToOldInterface());
1412 switch (base_header) {1413 switch (base_header) {
1413 .ofs_delta => |ofs_delta| {1414 .ofs_delta => |ofs_delta| {
1414 try delta_offsets.append(allocator, base_offset);1415 try delta_offsets.append(allocator, base_offset);
...@@ -1419,7 +1420,7 @@ fn indexPackHashDelta(...@@ -1419,7 +1420,7 @@ fn indexPackHashDelta(
1419 base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;1420 base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
1420 },1421 },
1421 else => {1422 else => {
1422 const base_data = try readObjectRaw(allocator, pack.deprecatedReader(), base_header.uncompressedLength());1423 const base_data = try readObjectRaw(allocator, &pack.interface, base_header.uncompressedLength());
1423 errdefer allocator.free(base_data);1424 errdefer allocator.free(base_data);
1424 const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };1425 const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
1425 try cache.put(allocator, base_offset, base_object);1426 try cache.put(allocator, base_offset, base_object);
...@@ -1444,7 +1445,7 @@ fn indexPackHashDelta(...@@ -1444,7 +1445,7 @@ fn indexPackHashDelta(
1444fn resolveDeltaChain(1445fn resolveDeltaChain(
1445 allocator: Allocator,1446 allocator: Allocator,
1446 format: Oid.Format,1447 format: Oid.Format,
1447 pack: std.fs.File,1448 pack: *std.fs.File.Reader,
1448 base_object: Object,1449 base_object: Object,
1449 delta_offsets: []const u64,1450 delta_offsets: []const u64,
1450 cache: *ObjectCache,1451 cache: *ObjectCache,
...@@ -1456,8 +1457,8 @@ fn resolveDeltaChain(...@@ -1456,8 +1457,8 @@ fn resolveDeltaChain(
14561457
1457 const delta_offset = delta_offsets[i];1458 const delta_offset = delta_offsets[i];
1458 try pack.seekTo(delta_offset);1459 try pack.seekTo(delta_offset);
1459 const delta_header = try EntryHeader.read(format, pack.deprecatedReader());1460 const delta_header = try EntryHeader.read(format, pack.interface.adaptToOldInterface());
1460 const delta_data = try readObjectRaw(allocator, pack.deprecatedReader(), delta_header.uncompressedLength());1461 const delta_data = try readObjectRaw(allocator, &pack.interface, delta_header.uncompressedLength());
1461 defer allocator.free(delta_data);1462 defer allocator.free(delta_data);
1462 var delta_stream = std.io.fixedBufferStream(delta_data);1463 var delta_stream = std.io.fixedBufferStream(delta_data);
1463 const delta_reader = delta_stream.reader();1464 const delta_reader = delta_stream.reader();
...@@ -1481,18 +1482,14 @@ fn resolveDeltaChain(...@@ -1481,18 +1482,14 @@ fn resolveDeltaChain(
1481/// Reads the complete contents of an object from `reader`. This function may1482/// Reads the complete contents of an object from `reader`. This function may
1482/// read more bytes than required from `reader`, so the reader position after1483/// read more bytes than required from `reader`, so the reader position after
1483/// returning is not reliable.1484/// returning is not reliable.
1484fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {1485fn readObjectRaw(allocator: Allocator, reader: *std.Io.Reader, size: u64) ![]u8 {
1485 const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;1486 const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
1486 var buffered_reader = std.io.bufferedReader(reader);1487 var aw: std.Io.Writer.Allocating = .init(allocator);
1487 var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader());1488 try aw.ensureTotalCapacity(alloc_size);
1488 const data = try allocator.alloc(u8, alloc_size);1489 defer aw.deinit();
1489 errdefer allocator.free(data);1490 var decompress: std.compress.flate.Decompress = .init(reader, .zlib, &.{});
1490 try decompress_stream.reader().readNoEof(data);1491 try decompress.reader.streamExact(&aw.writer, alloc_size);
1491 _ = decompress_stream.reader().readByte() catch |e| switch (e) {1492 return aw.toOwnedSlice();
1492 error.EndOfStream => return data,
1493 else => |other| return other,
1494 };
1495 return error.InvalidFormat;
1496}1493}
14971494
1498/// Expands delta data from `delta_reader` to `writer`. `base_object` must1495/// Expands delta data from `delta_reader` to `writer`. `base_object` must
src/link/Elf/Object.zig+7-8
...@@ -1198,15 +1198,14 @@ pub fn codeDecompressAlloc(self: *Object, elf_file: *Elf, atom_index: Atom.Index...@@ -1198,15 +1198,14 @@ pub fn codeDecompressAlloc(self: *Object, elf_file: *Elf, atom_index: Atom.Index
1198 const chdr = @as(*align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).*;1198 const chdr = @as(*align(1) const elf.Elf64_Chdr, @ptrCast(data.ptr)).*;
1199 switch (chdr.ch_type) {1199 switch (chdr.ch_type) {
1200 .ZLIB => {1200 .ZLIB => {
1201 var stream = std.io.fixedBufferStream(data[@sizeOf(elf.Elf64_Chdr)..]);1201 var stream: std.Io.Reader = .fixed(data[@sizeOf(elf.Elf64_Chdr)..]);
1202 var zlib_stream = std.compress.zlib.decompressor(stream.reader());1202 var zlib_stream: std.compress.flate.Decompress = .init(&stream, .zlib, &.{});
1203 const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;1203 const size = std.math.cast(usize, chdr.ch_size) orelse return error.Overflow;
1204 const decomp = try gpa.alloc(u8, size);1204 var aw: std.Io.Writer.Allocating = .init(gpa);
1205 const nread = zlib_stream.reader().readAll(decomp) catch return error.InputOutput;1205 try aw.ensureUnusedCapacity(size);
1206 if (nread != decomp.len) {1206 defer aw.deinit();
1207 return error.InputOutput;1207 _ = try zlib_stream.reader.streamRemaining(&aw.writer);
1208 }1208 return aw.toOwnedSlice();
1209 return decomp;
1210 },1209 },
1211 else => @panic("TODO unhandled compression scheme"),1210 else => @panic("TODO unhandled compression scheme"),
1212 }1211 }