authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-29 16:29:56-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-31 22:10:11-07:00
log3fff84a4a4fb8f9926436313d4e44773b8afc4eb
tree6b5d597c004fca46885b203e9441e31818ec45c2
parent6bcced31a04afeeead065af961f2571a95e4ad21

compiler: fix unit test compile errors

sorry, zip file creation has regressed because std lib no longer has a deflate compression implementation

4 files changed, 10 insertions(+), 372 deletions(-)

lib/std/zip.zig-4
......@@ -660,7 +660,3 @@ pub fn extract(dest: std.fs.Dir, fr: *File.Reader, options: ExtractOptions) !voi
660660 }
661661 }
662662}
663
664test {
665 _ = @import("zip/test.zig");
666}
lib/std/zip/test.zig deleted-298
......@@ -1,298 +0,0 @@
1const std = @import("std");
2const testing = std.testing;
3const zip = @import("../zip.zig");
4const maxInt = std.math.maxInt;
5
6pub const File = struct {
7 name: []const u8,
8 content: []const u8,
9 compression: zip.CompressionMethod,
10};
11
12pub fn expectFiles(
13 test_files: []const File,
14 dir: std.fs.Dir,
15 opt: struct {
16 strip_prefix: ?[]const u8 = null,
17 },
18) !void {
19 for (test_files) |test_file| {
20 var normalized_sub_path_buf: [std.fs.max_path_bytes]u8 = undefined;
21
22 const name = blk: {
23 if (opt.strip_prefix) |strip_prefix| {
24 try testing.expect(test_file.name.len >= strip_prefix.len);
25 try testing.expectEqualStrings(strip_prefix, test_file.name[0..strip_prefix.len]);
26 break :blk test_file.name[strip_prefix.len..];
27 }
28 break :blk test_file.name;
29 };
30 const normalized_sub_path = normalized_sub_path_buf[0..name.len];
31 @memcpy(normalized_sub_path, name);
32 std.mem.replaceScalar(u8, normalized_sub_path, '\\', '/');
33 var file = try dir.openFile(normalized_sub_path, .{});
34 defer file.close();
35 var content_buf: [4096]u8 = undefined;
36 const n = try file.deprecatedReader().readAll(&content_buf);
37 try testing.expectEqualStrings(test_file.content, content_buf[0..n]);
38 }
39}
40
41// Used to store any data from writing a file to the zip archive that's needed
42// when writing the corresponding central directory record.
43pub const FileStore = struct {
44 compression: zip.CompressionMethod,
45 file_offset: u64,
46 crc32: u32,
47 compressed_size: u32,
48 uncompressed_size: usize,
49};
50
51pub fn makeZip(
52 buf: []u8,
53 comptime files: []const File,
54 options: WriteZipOptions,
55) !std.io.FixedBufferStream([]u8) {
56 var store: [files.len]FileStore = undefined;
57 return try makeZipWithStore(buf, files, options, &store);
58}
59
60pub fn makeZipWithStore(
61 buf: []u8,
62 files: []const File,
63 options: WriteZipOptions,
64 store: []FileStore,
65) !std.io.FixedBufferStream([]u8) {
66 var fbs = std.io.fixedBufferStream(buf);
67 try writeZip(fbs.writer(), files, store, options);
68 return std.io.fixedBufferStream(buf[0..fbs.pos]);
69}
70
71pub const WriteZipOptions = struct {
72 end: ?EndRecordOptions = null,
73 local_header: ?LocalHeaderOptions = null,
74};
75pub const LocalHeaderOptions = struct {
76 zip64: ?LocalHeaderZip64Options = null,
77 compressed_size: ?u32 = null,
78 uncompressed_size: ?u32 = null,
79 extra_len: ?u16 = null,
80};
81pub const LocalHeaderZip64Options = struct {
82 data_size: ?u16 = null,
83};
84pub const EndRecordOptions = struct {
85 zip64: ?Zip64Options = null,
86 sig: ?[4]u8 = null,
87 disk_number: ?u16 = null,
88 central_directory_disk_number: ?u16 = null,
89 record_count_disk: ?u16 = null,
90 record_count_total: ?u16 = null,
91 central_directory_size: ?u32 = null,
92 central_directory_offset: ?u32 = null,
93 comment_len: ?u16 = null,
94 comment: ?[]const u8 = null,
95};
96pub const Zip64Options = struct {
97 locator_sig: ?[4]u8 = null,
98 locator_zip64_disk_count: ?u32 = null,
99 locator_record_file_offset: ?u64 = null,
100 locator_total_disk_count: ?u32 = null,
101 //record_size: ?u64 = null,
102 central_directory_size: ?u64 = null,
103};
104
105pub fn writeZip(
106 writer: anytype,
107 files: []const File,
108 store: []FileStore,
109 options: WriteZipOptions,
110) !void {
111 if (store.len < files.len) return error.FileStoreTooSmall;
112 var zipper = initZipper(writer);
113 for (files, 0..) |file, i| {
114 store[i] = try zipper.writeFile(.{
115 .name = file.name,
116 .content = file.content,
117 .compression = file.compression,
118 .write_options = options,
119 });
120 }
121 for (files, 0..) |file, i| {
122 try zipper.writeCentralRecord(store[i], .{
123 .name = file.name,
124 });
125 }
126 try zipper.writeEndRecord(if (options.end) |e| e else .{});
127}
128
129pub fn initZipper(writer: anytype) Zipper(@TypeOf(writer)) {
130 return .{ .counting_writer = std.io.countingWriter(writer) };
131}
132
133/// Provides methods to format and write the contents of a zip archive
134/// to the underlying Writer.
135pub fn Zipper(comptime Writer: type) type {
136 return struct {
137 counting_writer: std.io.CountingWriter(Writer),
138 central_count: u64 = 0,
139 first_central_offset: ?u64 = null,
140 last_central_limit: ?u64 = null,
141
142 const Self = @This();
143
144 pub fn writeFile(
145 self: *Self,
146 opt: struct {
147 name: []const u8,
148 content: []const u8,
149 compression: zip.CompressionMethod,
150 write_options: WriteZipOptions,
151 },
152 ) !FileStore {
153 const writer = self.counting_writer.writer();
154
155 const file_offset: u64 = @intCast(self.counting_writer.bytes_written);
156 const crc32 = std.hash.Crc32.hash(opt.content);
157
158 const header_options = opt.write_options.local_header;
159 {
160 var compressed_size: u32 = 0;
161 var uncompressed_size: u32 = 0;
162 var extra_len: u16 = 0;
163 if (header_options) |hdr_options| {
164 compressed_size = if (hdr_options.compressed_size) |size| size else 0;
165 uncompressed_size = if (hdr_options.uncompressed_size) |size| size else @intCast(opt.content.len);
166 extra_len = if (hdr_options.extra_len) |len| len else 0;
167 }
168 const hdr: zip.LocalFileHeader = .{
169 .signature = zip.local_file_header_sig,
170 .version_needed_to_extract = 10,
171 .flags = .{ .encrypted = false, ._ = 0 },
172 .compression_method = opt.compression,
173 .last_modification_time = 0,
174 .last_modification_date = 0,
175 .crc32 = crc32,
176 .compressed_size = compressed_size,
177 .uncompressed_size = uncompressed_size,
178 .filename_len = @intCast(opt.name.len),
179 .extra_len = extra_len,
180 };
181 try writer.writeStructEndian(hdr, .little);
182 }
183 try writer.writeAll(opt.name);
184
185 if (header_options) |hdr| {
186 if (hdr.zip64) |options| {
187 try writer.writeInt(u16, 0x0001, .little);
188 const data_size = if (options.data_size) |size| size else 8;
189 try writer.writeInt(u16, data_size, .little);
190 try writer.writeInt(u64, 0, .little);
191 try writer.writeInt(u64, @intCast(opt.content.len), .little);
192 }
193 }
194
195 var compressed_size: u32 = undefined;
196 switch (opt.compression) {
197 .store => {
198 try writer.writeAll(opt.content);
199 compressed_size = @intCast(opt.content.len);
200 },
201 .deflate => {
202 const offset = self.counting_writer.bytes_written;
203 var fbs = std.io.fixedBufferStream(opt.content);
204 try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
205 std.debug.assert(fbs.pos == opt.content.len);
206 compressed_size = @intCast(self.counting_writer.bytes_written - offset);
207 },
208 else => unreachable,
209 }
210 return .{
211 .compression = opt.compression,
212 .file_offset = file_offset,
213 .crc32 = crc32,
214 .compressed_size = compressed_size,
215 .uncompressed_size = opt.content.len,
216 };
217 }
218
219 pub fn writeCentralRecord(
220 self: *Self,
221 store: FileStore,
222 opt: struct {
223 name: []const u8,
224 version_needed_to_extract: u16 = 10,
225 },
226 ) !void {
227 if (self.first_central_offset == null) {
228 self.first_central_offset = self.counting_writer.bytes_written;
229 }
230 self.central_count += 1;
231
232 const hdr: zip.CentralDirectoryFileHeader = .{
233 .signature = zip.central_file_header_sig,
234 .version_made_by = 0,
235 .version_needed_to_extract = opt.version_needed_to_extract,
236 .flags = .{ .encrypted = false, ._ = 0 },
237 .compression_method = store.compression,
238 .last_modification_time = 0,
239 .last_modification_date = 0,
240 .crc32 = store.crc32,
241 .compressed_size = store.compressed_size,
242 .uncompressed_size = @intCast(store.uncompressed_size),
243 .filename_len = @intCast(opt.name.len),
244 .extra_len = 0,
245 .comment_len = 0,
246 .disk_number = 0,
247 .internal_file_attributes = 0,
248 .external_file_attributes = 0,
249 .local_file_header_offset = @intCast(store.file_offset),
250 };
251 try self.counting_writer.writer().writeStructEndian(hdr, .little);
252 try self.counting_writer.writer().writeAll(opt.name);
253 self.last_central_limit = self.counting_writer.bytes_written;
254 }
255
256 pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {
257 const cd_offset = self.first_central_offset orelse 0;
258 const cd_end = self.last_central_limit orelse 0;
259
260 if (opt.zip64) |zip64| {
261 const end64_off = cd_end;
262 const fixed: zip.EndRecord64 = .{
263 .signature = zip.end_record64_sig,
264 .end_record_size = @sizeOf(zip.EndRecord64) - 12,
265 .version_made_by = 0,
266 .version_needed_to_extract = 45,
267 .disk_number = 0,
268 .central_directory_disk_number = 0,
269 .record_count_disk = @intCast(self.central_count),
270 .record_count_total = @intCast(self.central_count),
271 .central_directory_size = @intCast(cd_end - cd_offset),
272 .central_directory_offset = @intCast(cd_offset),
273 };
274 try self.counting_writer.writer().writeStructEndian(fixed, .little);
275 const locator: zip.EndLocator64 = .{
276 .signature = if (zip64.locator_sig) |s| s else zip.end_locator64_sig,
277 .zip64_disk_count = if (zip64.locator_zip64_disk_count) |c| c else 0,
278 .record_file_offset = if (zip64.locator_record_file_offset) |o| o else @intCast(end64_off),
279 .total_disk_count = if (zip64.locator_total_disk_count) |c| c else 1,
280 };
281 try self.counting_writer.writer().writeStructEndian(locator, .little);
282 }
283 const hdr: zip.EndRecord = .{
284 .signature = if (opt.sig) |s| s else zip.end_record_sig,
285 .disk_number = if (opt.disk_number) |n| n else 0,
286 .central_directory_disk_number = if (opt.central_directory_disk_number) |n| n else 0,
287 .record_count_disk = if (opt.record_count_disk) |c| c else @intCast(self.central_count),
288 .record_count_total = if (opt.record_count_total) |c| c else @intCast(self.central_count),
289 .central_directory_size = if (opt.central_directory_size) |s| s else @intCast(cd_end - cd_offset),
290 .central_directory_offset = if (opt.central_directory_offset) |o| o else @intCast(cd_offset),
291 .comment_len = if (opt.comment_len) |l| l else (if (opt.comment) |c| @as(u16, @intCast(c.len)) else 0),
292 };
293 try self.counting_writer.writer().writeStructEndian(hdr, .little);
294 if (opt.comment) |c|
295 try self.counting_writer.writer().writeAll(c);
296 }
297 };
298}
src/Package/Fetch.zig-66
......@@ -2076,72 +2076,6 @@ const UnpackResult = struct {
20762076 }
20772077};
20782078
2079test "zip" {
2080 const gpa = std.testing.allocator;
2081 var tmp = std.testing.tmpDir(.{});
2082 defer tmp.cleanup();
2083
2084 const test_files = [_]std.zip.testutil.File{
2085 .{ .name = "foo", .content = "this is just foo\n", .compression = .store },
2086 .{ .name = "bar", .content = "another file\n", .compression = .deflate },
2087 };
2088 {
2089 var zip_file = try tmp.dir.createFile("test.zip", .{});
2090 defer zip_file.close();
2091 var bw = std.io.bufferedWriter(zip_file.deprecatedWriter());
2092 var store: [test_files.len]std.zip.testutil.FileStore = undefined;
2093 try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
2094 try bw.flush();
2095 }
2096
2097 const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
2098 defer gpa.free(zip_path);
2099
2100 var fb: TestFetchBuilder = undefined;
2101 var fetch = try fb.build(gpa, tmp.dir, zip_path);
2102 defer fb.deinit();
2103
2104 try fetch.run();
2105
2106 var out = try fb.packageDir();
2107 defer out.close();
2108
2109 try std.zip.testutil.expectFiles(&test_files, out, .{});
2110}
2111
2112test "zip with one root folder" {
2113 const gpa = std.testing.allocator;
2114 var tmp = std.testing.tmpDir(.{});
2115 defer tmp.cleanup();
2116
2117 const test_files = [_]std.zip.testutil.File{
2118 .{ .name = "the_root_folder/foo.zig", .content = "// this is foo.zig\n", .compression = .store },
2119 .{ .name = "the_root_folder/README.md", .content = "# The foo.zig README\n", .compression = .store },
2120 };
2121 {
2122 var zip_file = try tmp.dir.createFile("test.zip", .{});
2123 defer zip_file.close();
2124 var bw = std.io.bufferedWriter(zip_file.deprecatedWriter());
2125 var store: [test_files.len]std.zip.testutil.FileStore = undefined;
2126 try std.zip.testutil.writeZip(bw.writer(), &test_files, &store, .{});
2127 try bw.flush();
2128 }
2129
2130 const zip_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/test.zip", .{tmp.sub_path});
2131 defer gpa.free(zip_path);
2132
2133 var fb: TestFetchBuilder = undefined;
2134 var fetch = try fb.build(gpa, tmp.dir, zip_path);
2135 defer fb.deinit();
2136
2137 try fetch.run();
2138
2139 var out = try fb.packageDir();
2140 defer out.close();
2141
2142 try std.zip.testutil.expectFiles(&test_files, out, .{ .strip_prefix = "the_root_folder/" });
2143}
2144
21452079test "tarball with duplicate paths" {
21462080 // This tarball has duplicate path 'dir1/file1' to simulate case sensitve
21472081 // file system on any file sytstem.
src/Package/Fetch/git.zig+10-4
......@@ -1564,9 +1564,12 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
15641564 defer pack_file.close();
15651565 try pack_file.writeAll(testrepo_pack);
15661566
1567 var pack_file_buffer: [4096]u8 = undefined;
1568 var pack_file_reader = pack_file.reader(&pack_file_buffer);
1569
15671570 var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
15681571 defer index_file.close();
1569 try indexPack(testing.allocator, format, pack_file, index_file.deprecatedWriter());
1572 try indexPack(testing.allocator, format, &pack_file_reader, index_file.deprecatedWriter());
15701573
15711574 // Arbitrary size limit on files read while checking the repository contents
15721575 // (all files in the test repo are known to be smaller than this)
......@@ -1580,7 +1583,7 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
15801583 const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
15811584 try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
15821585
1583 var repository = try Repository.init(testing.allocator, format, pack_file, index_file);
1586 var repository = try Repository.init(testing.allocator, format, &pack_file_reader, index_file);
15841587 defer repository.deinit();
15851588
15861589 var worktree = testing.tmpDir(.{ .iterate = true });
......@@ -1673,6 +1676,9 @@ pub fn main() !void {
16731676
16741677 var pack_file = try std.fs.cwd().openFile(args[2], .{});
16751678 defer pack_file.close();
1679 var pack_file_buffer: [4096]u8 = undefined;
1680 var pack_file_reader = pack_file.reader(&pack_file_buffer);
1681
16761682 const commit = try Oid.parse(format, args[3]);
16771683 var worktree = try std.fs.cwd().makeOpenPath(args[4], .{});
16781684 defer worktree.close();
......@@ -1684,11 +1690,11 @@ pub fn main() !void {
16841690 var index_file = try git_dir.createFile("idx", .{ .read = true });
16851691 defer index_file.close();
16861692 var index_buffered_writer = std.io.bufferedWriter(index_file.deprecatedWriter());
1687 try indexPack(allocator, format, pack_file, index_buffered_writer.writer());
1693 try indexPack(allocator, format, &pack_file_reader, index_buffered_writer.writer());
16881694 try index_buffered_writer.flush();
16891695
16901696 std.debug.print("Starting checkout...\n", .{});
1691 var repository = try Repository.init(allocator, format, pack_file, index_file);
1697 var repository = try Repository.init(allocator, format, &pack_file_reader, index_file);
16921698 defer repository.deinit();
16931699 var diagnostics: Diagnostics = .{ .allocator = allocator };
16941700 defer diagnostics.deinit();