authorgravatar for thatlemon@gmail.comLemonBoy <thatlemon@gmail.com> 2020-09-11 22:17:08+02:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-11 20:02:41-04:00
log9fe4c89230df2d78c8bf37b4b1d7a9bedb92677b
treec90545c09f7f52cb19d3fafb301f10acc10b5544
parent1edf097871aaed4926d0107b86960c4a3f7e1287

std: Add a gzip decoder

4 files changed, 251 insertions(+), 0 deletions(-)

build.zig+1
......@@ -128,6 +128,7 @@ pub fn build(b: *Builder) !void {
128128 "README.md",
129129 ".z.0",
130130 ".z.9",
131 ".gz",
131132 "rfc1951.txt",
132133 },
133134 });
lib/std/compress.zig+2
......@@ -6,8 +6,10 @@
66const std = @import("std.zig");
77
88pub const deflate = @import("compress/deflate.zig");
9pub const gzip = @import("compress/gzip.zig");
910pub const zlib = @import("compress/zlib.zig");
1011
1112test "" {
13 _ = gzip;
1214 _ = zlib;
1315}
lib/std/compress/gzip.zig created+248
......@@ -0,0 +1,248 @@
1// SPDX-License-Identifier: MIT
2// Copyright (c) 2015-2020 Zig Contributors
3// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.
6//
7// Decompressor for GZIP data streams (RFC1952)
8
9const std = @import("std");
10const io = std.io;
11const fs = std.fs;
12const testing = std.testing;
13const mem = std.mem;
14const deflate = std.compress.deflate;
15
16// Flags for the FLG field in the header
17const FTEXT = 1 << 0;
18const FHCRC = 1 << 1;
19const FEXTRA = 1 << 2;
20const FNAME = 1 << 3;
21const FCOMMENT = 1 << 4;
22
23pub fn GzipStream(comptime ReaderType: type) type {
24 return struct {
25 const Self = @This();
26
27 pub const Error = ReaderType.Error ||
28 deflate.InflateStream(ReaderType).Error ||
29 error{ CorruptedData, WrongChecksum };
30 pub const Reader = io.Reader(*Self, Error, read);
31
32 allocator: *mem.Allocator,
33 inflater: deflate.InflateStream(ReaderType),
34 in_reader: ReaderType,
35 hasher: std.hash.Crc32,
36 window_slice: []u8,
37 read_amt: usize,
38
39 info: struct {
40 filename: ?[]const u8,
41 comment: ?[]const u8,
42 modification_time: u32,
43 },
44
45 fn init(allocator: *mem.Allocator, source: ReaderType) !Self {
46 // gzip header format is specified in RFC1952
47 const header = try source.readBytesNoEof(10);
48
49 // Check the ID1/ID2 fields
50 if (header[0] != 0x1f or header[1] != 0x8b)
51 return error.BadHeader;
52
53 const CM = header[2];
54 // The CM field must be 8 to indicate the use of DEFLATE
55 if (CM != 8) return error.InvalidCompression;
56 // Flags
57 const FLG = header[3];
58 // Modification time, as a Unix timestamp.
59 // If zero there's no timestamp available.
60 const MTIME = mem.readIntLittle(u32, header[4..8]);
61 // Extra flags
62 const XFL = header[8];
63 // Operating system where the compression took place
64 const OS = header[9];
65
66 if (FLG & FEXTRA != 0) {
67 // Skip the extra data, we could read and expose it to the user
68 // if somebody needs it.
69 const len = try source.readIntLittle(u16);
70 try source.skipBytes(len, .{});
71 }
72
73 var filename: ?[]const u8 = null;
74 if (FLG & FNAME != 0) {
75 filename = try source.readUntilDelimiterAlloc(
76 allocator,
77 0,
78 std.math.maxInt(usize),
79 );
80 }
81 errdefer if (filename) |p| allocator.free(p);
82
83 var comment: ?[]const u8 = null;
84 if (FLG & FCOMMENT != 0) {
85 comment = try source.readUntilDelimiterAlloc(
86 allocator,
87 0,
88 std.math.maxInt(usize),
89 );
90 }
91 errdefer if (comment) |p| allocator.free(p);
92
93 if (FLG & FHCRC != 0) {
94 // TODO: Evaluate and check the header checksum. The stdlib has
95 // no CRC16 yet :(
96 _ = try source.readIntLittle(u16);
97 }
98
99 // The RFC doesn't say anything about the DEFLATE window size to be
100 // used, default to 32K.
101 var window_slice = try allocator.alloc(u8, 32 * 1024);
102
103 return Self{
104 .allocator = allocator,
105 .inflater = deflate.inflateStream(source, window_slice),
106 .in_reader = source,
107 .hasher = std.hash.Crc32.init(),
108 .window_slice = window_slice,
109 .info = .{
110 .filename = filename,
111 .comment = comment,
112 .modification_time = MTIME,
113 },
114 .read_amt = 0,
115 };
116 }
117
118 pub fn deinit(self: *Self) void {
119 self.allocator.free(self.window_slice);
120 if (self.info.filename) |filename|
121 self.allocator.free(filename);
122 if (self.info.comment) |comment|
123 self.allocator.free(comment);
124 }
125
126 // Implements the io.Reader interface
127 pub fn read(self: *Self, buffer: []u8) Error!usize {
128 if (buffer.len == 0)
129 return 0;
130
131 // Read from the compressed stream and update the computed checksum
132 const r = try self.inflater.read(buffer);
133 if (r != 0) {
134 self.hasher.update(buffer[0..r]);
135 self.read_amt += r;
136 return r;
137 }
138
139 // We've reached the end of stream, check if the checksum matches
140 const hash = try self.in_reader.readIntLittle(u32);
141 if (hash != self.hasher.final())
142 return error.WrongChecksum;
143
144 // The ISIZE field is the size of the uncompressed input modulo 2^32
145 const input_size = try self.in_reader.readIntLittle(u32);
146 if (self.read_amt & 0xffffffff != input_size)
147 return error.CorruptedData;
148
149 return 0;
150 }
151
152 pub fn reader(self: *Self) Reader {
153 return .{ .context = self };
154 }
155 };
156}
157
158pub fn gzipStream(allocator: *mem.Allocator, reader: anytype) !GzipStream(@TypeOf(reader)) {
159 return GzipStream(@TypeOf(reader)).init(allocator, reader);
160}
161
162fn testReader(data: []const u8, comptime expected: []const u8) !void {
163 var in_stream = io.fixedBufferStream(data);
164
165 var gzip_stream = try gzipStream(testing.allocator, in_stream.reader());
166 defer gzip_stream.deinit();
167
168 // Read and decompress the whole file
169 const buf = try gzip_stream.reader().readAllAlloc(testing.allocator, std.math.maxInt(usize));
170 defer testing.allocator.free(buf);
171 // Calculate its SHA256 hash and check it against the reference
172 var hash: [32]u8 = undefined;
173 std.crypto.hash.sha2.Sha256.hash(buf, hash[0..], .{});
174
175 assertEqual(expected, &hash);
176}
177
178// Assert `expected` == `input` where `input` is a bytestring.
179pub fn assertEqual(comptime expected: []const u8, input: []const u8) void {
180 var expected_bytes: [expected.len / 2]u8 = undefined;
181 for (expected_bytes) |*r, i| {
182 r.* = std.fmt.parseInt(u8, expected[2 * i .. 2 * i + 2], 16) catch unreachable;
183 }
184
185 testing.expectEqualSlices(u8, &expected_bytes, input);
186}
187
188// All the test cases are obtained by compressing the RFC1952 text
189//
190// https://tools.ietf.org/rfc/rfc1952.txt length=25037 bytes
191// SHA256=164ef0897b4cbec63abf1b57f069f3599bd0fb7c72c2a4dee21bd7e03ec9af67
192test "compressed data" {
193 try testReader(
194 @embedFile("rfc1952.txt.gz"),
195 "164ef0897b4cbec63abf1b57f069f3599bd0fb7c72c2a4dee21bd7e03ec9af67",
196 );
197}
198
199test "sanity checks" {
200 // Truncated header
201 testing.expectError(
202 error.EndOfStream,
203 testReader(&[_]u8{ 0x1f, 0x8B }, ""),
204 );
205 // Wrong CM
206 testing.expectError(
207 error.InvalidCompression,
208 testReader(&[_]u8{
209 0x1f, 0x8b, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00,
210 0x00, 0x03,
211 }, ""),
212 );
213 // Wrong checksum
214 testing.expectError(
215 error.WrongChecksum,
216 testReader(&[_]u8{
217 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
218 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x01,
219 0x00, 0x00, 0x00, 0x00,
220 }, ""),
221 );
222 // Truncated checksum
223 testing.expectError(
224 error.EndOfStream,
225 testReader(&[_]u8{
226 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
227 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00,
228 }, ""),
229 );
230 // Wrong initial size
231 testing.expectError(
232 error.CorruptedData,
233 testReader(&[_]u8{
234 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
235 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
236 0x00, 0x00, 0x00, 0x01,
237 }, ""),
238 );
239 // Truncated initial size field
240 testing.expectError(
241 error.EndOfStream,
242 testReader(&[_]u8{
243 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00,
244 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00,
245 0x00, 0x00, 0x00,
246 }, ""),
247 );
248}
lib/std/compress/rfc1952.txt.gz created
Binary files /dev/null and b/lib/std/compress/rfc1952.txt.gz differ