authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-22 14:14:26-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-08-26 21:00:58-07:00
log6464e0d4fc9937e154c34567891bae84c63732b9
treea42d9f4208bbea1dfab3f6e0f1d1b586fc79abb9
parentea0ce7afb59d7c7ed33f707f3fea3e0babd785b6

std.compress.xz: flatten namespaces

2 files changed, 207 insertions(+), 212 deletions(-)

lib/std/compress/xz.zig+207-4
...@@ -1,7 +1,10 @@...@@ -1,7 +1,10 @@
1const std = @import("std");1const std = @import("std");
2const block = @import("xz/block.zig");
3const Allocator = std.mem.Allocator;2const Allocator = std.mem.Allocator;
3const ArrayList = std.ArrayList;
4const Crc32 = std.hash.Crc32;4const Crc32 = std.hash.Crc32;
5const Crc64 = std.hash.crc.Crc64Xz;
6const Sha256 = std.crypto.hash.sha2.Sha256;
7const lzma2 = std.compress.lzma2;
58
6pub const Check = enum(u4) {9pub const Check = enum(u4) {
7 none = 0x00,10 none = 0x00,
...@@ -27,11 +30,11 @@ pub fn Decompress(comptime ReaderType: type) type {...@@ -27,11 +30,11 @@ pub fn Decompress(comptime ReaderType: type) type {
27 return struct {30 return struct {
28 const Self = @This();31 const Self = @This();
2932
30 pub const Error = ReaderType.Error || block.Decoder(ReaderType).Error;33 pub const Error = ReaderType.Error || Decoder(ReaderType).Error;
31 pub const Reader = std.io.GenericReader(*Self, Error, read);34 pub const Reader = std.io.GenericReader(*Self, Error, read);
3235
33 allocator: Allocator,36 allocator: Allocator,
34 block_decoder: block.Decoder(ReaderType),37 block_decoder: Decoder(ReaderType),
35 in_reader: ReaderType,38 in_reader: ReaderType,
3639
37 fn init(allocator: Allocator, source: ReaderType) !Self {40 fn init(allocator: Allocator, source: ReaderType) !Self {
...@@ -52,7 +55,7 @@ pub fn Decompress(comptime ReaderType: type) type {...@@ -52,7 +55,7 @@ pub fn Decompress(comptime ReaderType: type) type {
5255
53 return Self{56 return Self{
54 .allocator = allocator,57 .allocator = allocator,
55 .block_decoder = try block.decoder(allocator, source, check),58 .block_decoder = try decoder(allocator, source, check),
56 .in_reader = source,59 .in_reader = source,
57 };60 };
58 }61 }
...@@ -161,6 +164,206 @@ pub fn hashedReader(...@@ -161,6 +164,206 @@ pub fn hashedReader(
161 return .{ .child_reader = reader, .hasher = hasher };164 return .{ .child_reader = reader, .hasher = hasher };
162}165}
163166
167const DecodeError = error{
168 CorruptInput,
169 EndOfStream,
170 EndOfStreamWithNoError,
171 WrongChecksum,
172 Unsupported,
173 Overflow,
174};
175
176pub fn decoder(allocator: Allocator, reader: anytype, check: Check) !Decoder(@TypeOf(reader)) {
177 return Decoder(@TypeOf(reader)).init(allocator, reader, check);
178}
179
180pub fn Decoder(comptime ReaderType: type) type {
181 return struct {
182 const Self = @This();
183 pub const Error =
184 ReaderType.Error ||
185 DecodeError ||
186 Allocator.Error;
187 pub const Reader = std.io.GenericReader(*Self, Error, read);
188
189 allocator: Allocator,
190 inner_reader: ReaderType,
191 check: Check,
192 err: ?Error,
193 to_read: ArrayList(u8),
194 read_pos: usize,
195 block_count: usize,
196
197 fn init(allocator: Allocator, in_reader: ReaderType, check: Check) !Self {
198 return Self{
199 .allocator = allocator,
200 .inner_reader = in_reader,
201 .check = check,
202 .err = null,
203 .to_read = .{},
204 .read_pos = 0,
205 .block_count = 0,
206 };
207 }
208
209 pub fn deinit(self: *Self) void {
210 self.to_read.deinit(self.allocator);
211 }
212
213 pub fn reader(self: *Self) Reader {
214 return .{ .context = self };
215 }
216
217 pub fn read(self: *Self, output: []u8) Error!usize {
218 while (true) {
219 const unread_len = self.to_read.items.len - self.read_pos;
220 if (unread_len > 0) {
221 const n = @min(unread_len, output.len);
222 @memcpy(output[0..n], self.to_read.items[self.read_pos..][0..n]);
223 self.read_pos += n;
224 return n;
225 }
226 if (self.err) |e| {
227 if (e == DecodeError.EndOfStreamWithNoError) {
228 return 0;
229 }
230 return e;
231 }
232 if (self.read_pos > 0) {
233 self.to_read.shrinkRetainingCapacity(0);
234 self.read_pos = 0;
235 }
236 self.readBlock() catch |e| {
237 self.err = e;
238 };
239 }
240 }
241
242 fn readBlock(self: *Self) Error!void {
243 var block_counter = std.io.countingReader(self.inner_reader);
244 const block_reader = block_counter.reader();
245
246 var packed_size: ?u64 = null;
247 var unpacked_size: ?u64 = null;
248
249 // Block Header
250 {
251 var header_hasher = hashedReader(block_reader, Crc32.init());
252 const header_reader = header_hasher.reader();
253
254 const header_size = @as(u64, try header_reader.readByte()) * 4;
255 if (header_size == 0)
256 return error.EndOfStreamWithNoError;
257
258 const Flags = packed struct(u8) {
259 last_filter_index: u2,
260 reserved: u4,
261 has_packed_size: bool,
262 has_unpacked_size: bool,
263 };
264
265 const flags = @as(Flags, @bitCast(try header_reader.readByte()));
266 const filter_count = @as(u3, flags.last_filter_index) + 1;
267 if (filter_count > 1)
268 return error.Unsupported;
269
270 if (flags.has_packed_size)
271 packed_size = try std.leb.readUleb128(u64, header_reader);
272
273 if (flags.has_unpacked_size)
274 unpacked_size = try std.leb.readUleb128(u64, header_reader);
275
276 const FilterId = enum(u64) {
277 lzma2 = 0x21,
278 _,
279 };
280
281 const filter_id = @as(
282 FilterId,
283 @enumFromInt(try std.leb.readUleb128(u64, header_reader)),
284 );
285
286 if (@intFromEnum(filter_id) >= 0x4000_0000_0000_0000)
287 return error.CorruptInput;
288
289 if (filter_id != .lzma2)
290 return error.Unsupported;
291
292 const properties_size = try std.leb.readUleb128(u64, header_reader);
293 if (properties_size != 1)
294 return error.CorruptInput;
295
296 // TODO: use filter properties
297 _ = try header_reader.readByte();
298
299 while (block_counter.bytes_read != header_size) {
300 if (try header_reader.readByte() != 0)
301 return error.CorruptInput;
302 }
303
304 const hash_a = header_hasher.hasher.final();
305 const hash_b = try header_reader.readInt(u32, .little);
306 if (hash_a != hash_b)
307 return error.WrongChecksum;
308 }
309
310 // Compressed Data
311 var packed_counter = std.io.countingReader(block_reader);
312 try lzma2.decompress(
313 self.allocator,
314 packed_counter.reader(),
315 self.to_read.writer(self.allocator),
316 );
317
318 if (packed_size) |s| {
319 if (s != packed_counter.bytes_read)
320 return error.CorruptInput;
321 }
322
323 const unpacked_bytes = self.to_read.items;
324 if (unpacked_size) |s| {
325 if (s != unpacked_bytes.len)
326 return error.CorruptInput;
327 }
328
329 // Block Padding
330 while (block_counter.bytes_read % 4 != 0) {
331 if (try block_reader.readByte() != 0)
332 return error.CorruptInput;
333 }
334
335 switch (self.check) {
336 .none => {},
337 .crc32 => {
338 const hash_a = Crc32.hash(unpacked_bytes);
339 const hash_b = try self.inner_reader.readInt(u32, .little);
340 if (hash_a != hash_b)
341 return error.WrongChecksum;
342 },
343 .crc64 => {
344 const hash_a = Crc64.hash(unpacked_bytes);
345 const hash_b = try self.inner_reader.readInt(u64, .little);
346 if (hash_a != hash_b)
347 return error.WrongChecksum;
348 },
349 .sha256 => {
350 var hash_a: [Sha256.digest_length]u8 = undefined;
351 Sha256.hash(unpacked_bytes, &hash_a, .{});
352
353 var hash_b: [Sha256.digest_length]u8 = undefined;
354 try self.inner_reader.readNoEof(&hash_b);
355
356 if (!std.mem.eql(u8, &hash_a, &hash_b))
357 return error.WrongChecksum;
358 },
359 else => return error.Unsupported,
360 }
361
362 self.block_count += 1;
363 }
364 };
365}
366
164test {367test {
165 _ = @import("xz/test.zig");368 _ = @import("xz/test.zig");
166}369}
lib/std/compress/xz/block.zig deleted-208
...@@ -1,208 +0,0 @@
1const std = @import("../../std.zig");
2const lzma2 = std.compress.lzma2;
3const Allocator = std.mem.Allocator;
4const ArrayListUnmanaged = std.ArrayListUnmanaged;
5const Crc32 = std.hash.Crc32;
6const Crc64 = std.hash.crc.Crc64Xz;
7const Sha256 = std.crypto.hash.sha2.Sha256;
8const xz = std.compress.xz;
9
10const DecodeError = error{
11 CorruptInput,
12 EndOfStream,
13 EndOfStreamWithNoError,
14 WrongChecksum,
15 Unsupported,
16 Overflow,
17};
18
19pub fn decoder(allocator: Allocator, reader: anytype, check: xz.Check) !Decoder(@TypeOf(reader)) {
20 return Decoder(@TypeOf(reader)).init(allocator, reader, check);
21}
22
23pub fn Decoder(comptime ReaderType: type) type {
24 return struct {
25 const Self = @This();
26 pub const Error =
27 ReaderType.Error ||
28 DecodeError ||
29 Allocator.Error;
30 pub const Reader = std.io.GenericReader(*Self, Error, read);
31
32 allocator: Allocator,
33 inner_reader: ReaderType,
34 check: xz.Check,
35 err: ?Error,
36 to_read: ArrayListUnmanaged(u8),
37 read_pos: usize,
38 block_count: usize,
39
40 fn init(allocator: Allocator, in_reader: ReaderType, check: xz.Check) !Self {
41 return Self{
42 .allocator = allocator,
43 .inner_reader = in_reader,
44 .check = check,
45 .err = null,
46 .to_read = .{},
47 .read_pos = 0,
48 .block_count = 0,
49 };
50 }
51
52 pub fn deinit(self: *Self) void {
53 self.to_read.deinit(self.allocator);
54 }
55
56 pub fn reader(self: *Self) Reader {
57 return .{ .context = self };
58 }
59
60 pub fn read(self: *Self, output: []u8) Error!usize {
61 while (true) {
62 const unread_len = self.to_read.items.len - self.read_pos;
63 if (unread_len > 0) {
64 const n = @min(unread_len, output.len);
65 @memcpy(output[0..n], self.to_read.items[self.read_pos..][0..n]);
66 self.read_pos += n;
67 return n;
68 }
69 if (self.err) |e| {
70 if (e == DecodeError.EndOfStreamWithNoError) {
71 return 0;
72 }
73 return e;
74 }
75 if (self.read_pos > 0) {
76 self.to_read.shrinkRetainingCapacity(0);
77 self.read_pos = 0;
78 }
79 self.readBlock() catch |e| {
80 self.err = e;
81 };
82 }
83 }
84
85 fn readBlock(self: *Self) Error!void {
86 var block_counter = std.io.countingReader(self.inner_reader);
87 const block_reader = block_counter.reader();
88
89 var packed_size: ?u64 = null;
90 var unpacked_size: ?u64 = null;
91
92 // Block Header
93 {
94 var header_hasher = xz.hashedReader(block_reader, Crc32.init());
95 const header_reader = header_hasher.reader();
96
97 const header_size = @as(u64, try header_reader.readByte()) * 4;
98 if (header_size == 0)
99 return error.EndOfStreamWithNoError;
100
101 const Flags = packed struct(u8) {
102 last_filter_index: u2,
103 reserved: u4,
104 has_packed_size: bool,
105 has_unpacked_size: bool,
106 };
107
108 const flags = @as(Flags, @bitCast(try header_reader.readByte()));
109 const filter_count = @as(u3, flags.last_filter_index) + 1;
110 if (filter_count > 1)
111 return error.Unsupported;
112
113 if (flags.has_packed_size)
114 packed_size = try std.leb.readUleb128(u64, header_reader);
115
116 if (flags.has_unpacked_size)
117 unpacked_size = try std.leb.readUleb128(u64, header_reader);
118
119 const FilterId = enum(u64) {
120 lzma2 = 0x21,
121 _,
122 };
123
124 const filter_id = @as(
125 FilterId,
126 @enumFromInt(try std.leb.readUleb128(u64, header_reader)),
127 );
128
129 if (@intFromEnum(filter_id) >= 0x4000_0000_0000_0000)
130 return error.CorruptInput;
131
132 if (filter_id != .lzma2)
133 return error.Unsupported;
134
135 const properties_size = try std.leb.readUleb128(u64, header_reader);
136 if (properties_size != 1)
137 return error.CorruptInput;
138
139 // TODO: use filter properties
140 _ = try header_reader.readByte();
141
142 while (block_counter.bytes_read != header_size) {
143 if (try header_reader.readByte() != 0)
144 return error.CorruptInput;
145 }
146
147 const hash_a = header_hasher.hasher.final();
148 const hash_b = try header_reader.readInt(u32, .little);
149 if (hash_a != hash_b)
150 return error.WrongChecksum;
151 }
152
153 // Compressed Data
154 var packed_counter = std.io.countingReader(block_reader);
155 try lzma2.decompress(
156 self.allocator,
157 packed_counter.reader(),
158 self.to_read.writer(self.allocator),
159 );
160
161 if (packed_size) |s| {
162 if (s != packed_counter.bytes_read)
163 return error.CorruptInput;
164 }
165
166 const unpacked_bytes = self.to_read.items;
167 if (unpacked_size) |s| {
168 if (s != unpacked_bytes.len)
169 return error.CorruptInput;
170 }
171
172 // Block Padding
173 while (block_counter.bytes_read % 4 != 0) {
174 if (try block_reader.readByte() != 0)
175 return error.CorruptInput;
176 }
177
178 switch (self.check) {
179 .none => {},
180 .crc32 => {
181 const hash_a = Crc32.hash(unpacked_bytes);
182 const hash_b = try self.inner_reader.readInt(u32, .little);
183 if (hash_a != hash_b)
184 return error.WrongChecksum;
185 },
186 .crc64 => {
187 const hash_a = Crc64.hash(unpacked_bytes);
188 const hash_b = try self.inner_reader.readInt(u64, .little);
189 if (hash_a != hash_b)
190 return error.WrongChecksum;
191 },
192 .sha256 => {
193 var hash_a: [Sha256.digest_length]u8 = undefined;
194 Sha256.hash(unpacked_bytes, &hash_a, .{});
195
196 var hash_b: [Sha256.digest_length]u8 = undefined;
197 try self.inner_reader.readNoEof(&hash_b);
198
199 if (!std.mem.eql(u8, &hash_a, &hash_b))
200 return error.WrongChecksum;
201 },
202 else => return error.Unsupported,
203 }
204
205 self.block_count += 1;
206 }
207 };
208}