authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-04-09 19:58:21-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-07-01 16:35:26-07:00
log6ac7931bec29f1cd4c889d6913a21f28e17c13a8
treea91d44087acb1c8886d9e92e6df555539db8b59f
parent890a02c3456dce7242aa65e5093b31f9d8a417bc

std: hacking around with buffered reader / writer semantics

I think I'm going to back out these vtable changes in the next commit

29 files changed, 2941 insertions(+), 2793 deletions(-)

lib/std/compress/flate.zig+14-41
...@@ -1,3 +1,5 @@...@@ -1,3 +1,5 @@
1const std = @import("../std.zig");
2
1/// Deflate is a lossless data compression file format that uses a combination3/// Deflate is a lossless data compression file format that uses a combination
2/// of LZ77 and Huffman coding.4/// of LZ77 and Huffman coding.
3pub const deflate = @import("flate/deflate.zig");5pub const deflate = @import("flate/deflate.zig");
...@@ -7,77 +9,48 @@ pub const deflate = @import("flate/deflate.zig");...@@ -7,77 +9,48 @@ pub const deflate = @import("flate/deflate.zig");
7pub const inflate = @import("flate/inflate.zig");9pub const inflate = @import("flate/inflate.zig");
810
9/// Decompress compressed data from reader and write plain data to the writer.11/// Decompress compressed data from reader and write plain data to the writer.
10pub fn decompress(reader: anytype, writer: anytype) !void {12pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) anyerror!void {
11 try inflate.decompress(.raw, reader, writer);13 try inflate.decompress(.raw, reader, writer);
12}14}
1315
14/// Decompressor type16pub const Decompressor = inflate.Decompressor(.raw);
15pub fn Decompressor(comptime ReaderType: type) type {
16 return inflate.Decompressor(.raw, ReaderType);
17}
18
19/// Create Decompressor which will read compressed data from reader.
20pub fn decompressor(reader: anytype) Decompressor(@TypeOf(reader)) {
21 return inflate.decompressor(.raw, reader);
22}
2317
24/// Compression level, trades between speed and compression size.18/// Compression level, trades between speed and compression size.
25pub const Options = deflate.Options;19pub const Options = deflate.Options;
2620
27/// Compress plain data from reader and write compressed data to the writer.21/// Compress plain data from reader and write compressed data to the writer.
28pub fn compress(reader: anytype, writer: anytype, options: Options) !void {22pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter, options: Options) anyerror!void {
29 try deflate.compress(.raw, reader, writer, options);23 try deflate.compress(.raw, reader, writer, options);
30}24}
3125
32/// Compressor type26pub const Compressor = deflate.Compressor(.raw);
33pub fn Compressor(comptime WriterType: type) type {
34 return deflate.Compressor(.raw, WriterType);
35}
36
37/// Create Compressor which outputs compressed data to the writer.
38pub fn compressor(writer: anytype, options: Options) !Compressor(@TypeOf(writer)) {
39 return try deflate.compressor(.raw, writer, options);
40}
4127
42/// Huffman only compression. Without Lempel-Ziv match searching. Faster28/// Huffman only compression. Without Lempel-Ziv match searching. Faster
43/// compression, less memory requirements but bigger compressed sizes.29/// compression, less memory requirements but bigger compressed sizes.
44pub const huffman = struct {30pub const huffman = struct {
45 pub fn compress(reader: anytype, writer: anytype) !void {31 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) anyerror!void {
46 try deflate.huffman.compress(.raw, reader, writer);32 try deflate.huffman.compress(.raw, reader, writer);
47 }33 }
4834
49 pub fn Compressor(comptime WriterType: type) type {35 pub const Compressor = deflate.huffman.Compressor(.raw);
50 return deflate.huffman.Compressor(.raw, WriterType);
51 }
52
53 pub fn compressor(writer: anytype) !huffman.Compressor(@TypeOf(writer)) {
54 return deflate.huffman.compressor(.raw, writer);
55 }
56};36};
5737
58// No compression store only. Compressed size is slightly bigger than plain.38// No compression store only. Compressed size is slightly bigger than plain.
59pub const store = struct {39pub const store = struct {
60 pub fn compress(reader: anytype, writer: anytype) !void {40 pub fn compress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) anyerror!void {
61 try deflate.store.compress(.raw, reader, writer);41 try deflate.store.compress(.raw, reader, writer);
62 }42 }
6343
64 pub fn Compressor(comptime WriterType: type) type {44 pub const Compressor = deflate.store.Compressor(.raw);
65 return deflate.store.Compressor(.raw, WriterType);
66 }
67
68 pub fn compressor(writer: anytype) !store.Compressor(@TypeOf(writer)) {
69 return deflate.store.compressor(.raw, writer);
70 }
71};45};
7246
73/// Container defines header/footer around deflate bit stream. Gzip and zlib47const builtin = @import("builtin");
74/// compression algorithms are containers around deflate bit stream body.
75const Container = @import("flate/container.zig").Container;
76const std = @import("std");
77const testing = std.testing;48const testing = std.testing;
78const fixedBufferStream = std.io.fixedBufferStream;49const fixedBufferStream = std.io.fixedBufferStream;
79const print = std.debug.print;50const print = std.debug.print;
80const builtin = @import("builtin");51/// Container defines header/footer around deflate bit stream. Gzip and zlib
52/// compression algorithms are containers around deflate bit stream body.
53const Container = @import("flate/container.zig").Container;
8154
82test {55test {
83 _ = deflate;56 _ = deflate;
lib/std/compress/flate/bit_reader.zig deleted-421
...@@ -1,421 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const testing = std.testing;
4
5pub const Flags = packed struct(u3) {
6 /// dont advance internal buffer, just get bits, leave them in buffer
7 peek: bool = false,
8 /// assume that there is no need to fill, fill should be called before
9 buffered: bool = false,
10 /// bit reverse read bits
11 reverse: bool = false,
12};
13
14/// Bit reader used during inflate (decompression). Has internal buffer of 64
15/// bits which shifts right after bits are consumed. Uses forward_reader to fill
16/// that internal buffer when needed.
17///
18/// readF is the core function. Supports few different ways of getting bits
19/// controlled by flags. In hot path we try to avoid checking whether we need to
20/// fill buffer from forward_reader by calling fill in advance and readF with
21/// buffered flag set.
22///
23pub fn BitReader(comptime T: type) type {
24 assert(T == u32 or T == u64);
25 const t_bytes: usize = @sizeOf(T);
26 const Tshift = if (T == u64) u6 else u5;
27
28 return struct {
29 // Underlying reader used for filling internal bits buffer
30 forward_reader: *std.io.BufferedReader,
31 // Internal buffer of 64 bits
32 bits: T = 0,
33 // Number of bits in the buffer
34 nbits: u32 = 0,
35
36 const Self = @This();
37
38 pub fn init(forward_reader: *std.io.BufferedReader) Self {
39 var self = Self{ .forward_reader = forward_reader };
40 self.fill(1) catch {};
41 return self;
42 }
43
44 /// Try to have `nice` bits are available in buffer. Reads from
45 /// forward reader if there is no `nice` bits in buffer. Returns error
46 /// if end of forward stream is reached and internal buffer is empty.
47 /// It will not error if less than `nice` bits are in buffer, only when
48 /// all bits are exhausted. During inflate we usually know what is the
49 /// maximum bits for the next step but usually that step will need less
50 /// bits to decode. So `nice` is not hard limit, it will just try to have
51 /// that number of bits available. If end of forward stream is reached
52 /// it may be some extra zero bits in buffer.
53 pub fn fill(self: *Self, nice: u6) !void {
54 if (self.nbits >= nice and nice != 0) {
55 return; // We have enough bits
56 }
57 // Read more bits from forward reader
58
59 // Number of empty bytes in bits, round nbits to whole bytes.
60 const empty_bytes =
61 @as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
62 (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
63
64 var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
65 const bytes_read = self.forward_reader.readAll(buf[0..empty_bytes]) catch 0;
66 if (bytes_read > 0) {
67 const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
68 self.bits |= u << @as(Tshift, @intCast(self.nbits));
69 self.nbits += 8 * @as(u8, @intCast(bytes_read));
70 return;
71 }
72
73 if (self.nbits == 0)
74 return error.EndOfStream;
75 }
76
77 /// Read exactly buf.len bytes into buf.
78 pub fn readAll(self: *Self, buf: []u8) !void {
79 assert(self.alignBits() == 0); // internal bits must be at byte boundary
80
81 // First read from internal bits buffer.
82 var n: usize = 0;
83 while (self.nbits > 0 and n < buf.len) {
84 buf[n] = try self.readF(u8, .{ .buffered = true });
85 n += 1;
86 }
87 // Then use forward reader for all other bytes.
88 try self.forward_reader.readNoEof(buf[n..]);
89 }
90
91 /// Alias for readF(U, 0).
92 pub fn read(self: *Self, comptime U: type) !U {
93 return self.readF(U, 0);
94 }
95
96 /// Alias for readF with flag.peak set.
97 pub inline fn peekF(self: *Self, comptime U: type, comptime how: Flags) !U {
98 return self.readF(U, .{
99 .peek = true,
100 .buffered = how.buffered,
101 .reverse = how.reverse,
102 });
103 }
104
105 /// Read with flags provided.
106 pub fn readF(self: *Self, comptime U: type, comptime how: Flags) !U {
107 if (U == T) {
108 assert(how == 0);
109 assert(self.alignBits() == 0);
110 try self.fill(@bitSizeOf(T));
111 if (self.nbits != @bitSizeOf(T)) return error.EndOfStream;
112 const v = self.bits;
113 self.nbits = 0;
114 self.bits = 0;
115 return v;
116 }
117 const n: Tshift = @bitSizeOf(U);
118 switch (how) {
119 0 => { // `normal` read
120 try self.fill(n); // ensure that there are n bits in the buffer
121 const u: U = @truncate(self.bits); // get n bits
122 try self.shift(n); // advance buffer for n
123 return u;
124 },
125 .{ .peek = true } => { // no shift, leave bits in the buffer
126 try self.fill(n);
127 return @truncate(self.bits);
128 },
129 .{ .buffered = true } => { // no fill, assume that buffer has enough bits
130 const u: U = @truncate(self.bits);
131 try self.shift(n);
132 return u;
133 },
134 .{ .reverse = true } => { // same as 0 with bit reverse
135 try self.fill(n);
136 const u: U = @truncate(self.bits);
137 try self.shift(n);
138 return @bitReverse(u);
139 },
140 .{ .peek = true, .reverse = true } => {
141 try self.fill(n);
142 return @bitReverse(@as(U, @truncate(self.bits)));
143 },
144 .{ .buffered = true, .reverse = true } => {
145 const u: U = @truncate(self.bits);
146 try self.shift(n);
147 return @bitReverse(u);
148 },
149 .{ .peek = true, .buffered = true },
150 => {
151 return @truncate(self.bits);
152 },
153 .{ .peek = true, .buffered = true, .reverse = true } => {
154 return @bitReverse(@as(U, @truncate(self.bits)));
155 },
156 }
157 }
158
159 /// Read n number of bits.
160 /// Only buffered flag can be used in how.
161 pub fn readN(self: *Self, n: u4, comptime how: u3) !u16 {
162 switch (how) {
163 0 => {
164 try self.fill(n);
165 },
166 .{ .buffered = true } => {},
167 else => unreachable,
168 }
169 const mask: u16 = (@as(u16, 1) << n) - 1;
170 const u: u16 = @as(u16, @truncate(self.bits)) & mask;
171 try self.shift(n);
172 return u;
173 }
174
175 /// Advance buffer for n bits.
176 pub fn shift(self: *Self, n: Tshift) !void {
177 if (n > self.nbits) return error.EndOfStream;
178 self.bits >>= n;
179 self.nbits -= n;
180 }
181
182 /// Skip n bytes.
183 pub fn skipBytes(self: *Self, n: u16) !void {
184 for (0..n) |_| {
185 try self.fill(8);
186 try self.shift(8);
187 }
188 }
189
190 // Number of bits to align stream to the byte boundary.
191 fn alignBits(self: *Self) u3 {
192 return @intCast(self.nbits & 0x7);
193 }
194
195 /// Align stream to the byte boundary.
196 pub fn alignToByte(self: *Self) void {
197 const ab = self.alignBits();
198 if (ab > 0) self.shift(ab) catch unreachable;
199 }
200
201 /// Skip zero terminated string.
202 pub fn skipStringZ(self: *Self) !void {
203 while (true) {
204 if (try self.readF(u8, 0) == 0) break;
205 }
206 }
207
208 /// Read deflate fixed fixed code.
209 /// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
210 /// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
211 /// Lit Value Bits Codes
212 /// --------- ---- -----
213 /// 0 - 143 8 00110000 through
214 /// 10111111
215 /// 144 - 255 9 110010000 through
216 /// 111111111
217 /// 256 - 279 7 0000000 through
218 /// 0010111
219 /// 280 - 287 8 11000000 through
220 /// 11000111
221 pub fn readFixedCode(self: *Self) !u16 {
222 try self.fill(7 + 2);
223 const code7 = try self.readF(u7, .{ .buffered = true, .reverse = true });
224 if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
225 return @as(u16, code7) + 256;
226 } else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
227 return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, .{ .buffered = true })) - 0b0011_0000;
228 } else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
229 return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, .{ .buffered = true }) + 280;
230 } else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
231 return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, .{ .buffered = true, .reverse = true })) + 144;
232 }
233 }
234 };
235}
236
237test "readF" {
238 var input: std.io.BufferedReader = undefined;
239 input.initFixed(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
240 var br: BitReader(u64) = .init(&input);
241
242 try testing.expectEqual(@as(u8, 48), br.nbits);
243 try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
244
245 try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
246 try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
247 try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
248 try testing.expectEqual(@as(u3, 5), br.alignBits());
249
250 try testing.expect(try br.readF(u8, .{ .peek = true }) == 0b0001_1110);
251 try testing.expect(try br.readF(u9, .{ .peek = true }) == 0b1_0001_1110);
252 try br.shift(9);
253 try testing.expectEqual(@as(u8, 36), br.nbits);
254 try testing.expectEqual(@as(u3, 4), br.alignBits());
255
256 try testing.expect(try br.readF(u4, 0) == 0b0100);
257 try testing.expectEqual(@as(u8, 32), br.nbits);
258 try testing.expectEqual(@as(u3, 0), br.alignBits());
259
260 try br.shift(1);
261 try testing.expectEqual(@as(u3, 7), br.alignBits());
262 try br.shift(1);
263 try testing.expectEqual(@as(u3, 6), br.alignBits());
264 br.alignToByte();
265 try testing.expectEqual(@as(u3, 0), br.alignBits());
266
267 try testing.expectEqual(@as(u64, 0xc9), br.bits);
268 try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
269 try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
270}
271
272test "read block type 1 data" {
273 inline for ([_]type{ u64, u32 }) |T| {
274 const data = [_]u8{
275 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
276 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
277 0x0c, 0x01, 0x02, 0x03, //
278 0xaa, 0xbb, 0xcc, 0xdd,
279 };
280 var fbs: std.io.BufferedReader = undefined;
281 fbs.initFixed(&data);
282 var br: BitReader(T) = .init(&fbs);
283
284 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
285 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
286
287 for ("Hello world\n") |c| {
288 try testing.expectEqual(@as(u8, c), try br.readF(u8, .{ .reverse = true }) - 0x30);
289 }
290 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
291 br.alignToByte();
292 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
293 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
294 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
295 }
296}
297
298test "shift/fill" {
299 const data = [_]u8{
300 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
301 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
302 };
303 var fbs: std.io.BufferedReader = undefined;
304 fbs.initFixed(&data);
305 var br: BitReader(u64) = .init(&fbs);
306
307 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
308 try br.shift(8);
309 try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
310 try br.fill(60); // fill with 1 byte
311 try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
312 try br.shift(8 * 4 + 4);
313 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
314
315 try br.fill(60); // fill with 4 bytes (shift by 4)
316 try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
317 try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
318
319 try br.shift(@intCast(br.nbits)); // clear buffer
320 try br.fill(8); // refill with the rest of the bytes
321 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
322}
323
324test "readAll" {
325 inline for ([_]type{ u64, u32 }) |T| {
326 const data = [_]u8{
327 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
328 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
329 };
330 var fbs: std.io.BufferedReader = undefined;
331 fbs.initFixed(&data);
332 var br: BitReader(T) = .init(&fbs);
333
334 switch (T) {
335 u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
336 u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
337 else => unreachable,
338 }
339
340 var out: [16]u8 = undefined;
341 try br.readAll(out[0..]);
342 try testing.expect(br.nbits == 0);
343 try testing.expect(br.bits == 0);
344
345 try testing.expectEqualSlices(u8, data[0..16], &out);
346 }
347}
348
349test "readFixedCode" {
350 inline for ([_]type{ u64, u32 }) |T| {
351 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
352
353 var fbs: std.io.BufferedReader = undefined;
354 fbs.initFixed(&fixed_codes);
355 var rdr: BitReader(T) = .init(&fbs);
356
357 for (0..286) |c| {
358 try testing.expectEqual(c, try rdr.readFixedCode());
359 }
360 try testing.expect(rdr.nbits == 0);
361 }
362}
363
364test "u32 leaves no bits on u32 reads" {
365 const data = [_]u8{
366 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
367 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
368 };
369 var fbs: std.io.BufferedReader = undefined;
370 fbs.initFixed(&data);
371 var br: BitReader(u32) = .init(&fbs);
372
373 _ = try br.read(u3);
374 try testing.expectEqual(29, br.nbits);
375 br.alignToByte();
376 try testing.expectEqual(24, br.nbits);
377 try testing.expectEqual(0x04_03_02_01, try br.read(u32));
378 try testing.expectEqual(0, br.nbits);
379 try testing.expectEqual(0x08_07_06_05, try br.read(u32));
380 try testing.expectEqual(0, br.nbits);
381
382 _ = try br.read(u9);
383 try testing.expectEqual(23, br.nbits);
384 br.alignToByte();
385 try testing.expectEqual(16, br.nbits);
386 try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32));
387 try testing.expectEqual(0, br.nbits);
388}
389
390test "u64 need fill after alignToByte" {
391 const data = [_]u8{
392 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
393 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
394 };
395
396 // without fill
397 var fbs: std.io.BufferedReader = undefined;
398 fbs.initFixed(&data);
399 var br: BitReader(u64) = .init(&fbs);
400 _ = try br.read(u23);
401 try testing.expectEqual(41, br.nbits);
402 br.alignToByte();
403 try testing.expectEqual(40, br.nbits);
404 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
405 try testing.expectEqual(8, br.nbits);
406 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
407 try testing.expectEqual(32, br.nbits);
408
409 // fill after align ensures all bits filled
410 fbs.reset();
411 br = .init(&fbs);
412 _ = try br.read(u23);
413 try testing.expectEqual(41, br.nbits);
414 br.alignToByte();
415 try br.fill(0);
416 try testing.expectEqual(64, br.nbits);
417 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
418 try testing.expectEqual(32, br.nbits);
419 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
420 try testing.expectEqual(0, br.nbits);
421}
lib/std/compress/flate/inflate.zig+480-30
...@@ -3,7 +3,6 @@ const assert = std.debug.assert;...@@ -3,7 +3,6 @@ const assert = std.debug.assert;
3const testing = std.testing;3const testing = std.testing;
44
5const hfd = @import("huffman_decoder.zig");5const hfd = @import("huffman_decoder.zig");
6const BitReader = @import("bit_reader.zig").BitReader;
7const CircularBuffer = @import("CircularBuffer.zig");6const CircularBuffer = @import("CircularBuffer.zig");
8const Container = @import("container.zig").Container;7const Container = @import("container.zig").Container;
9const Token = @import("Token.zig");8const Token = @import("Token.zig");
...@@ -48,16 +47,14 @@ pub fn Decompressor(comptime container: Container) type {...@@ -48,16 +47,14 @@ pub fn Decompressor(comptime container: Container) type {
48/// * 64K for history (CircularBuffer)47/// * 64K for history (CircularBuffer)
49/// * ~10K huffman decoders (Literal and DistanceDecoder)48/// * ~10K huffman decoders (Literal and DistanceDecoder)
50///49///
51pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type {50pub fn Inflate(comptime container: Container, comptime Lookahead: type) type {
52 assert(LookaheadType == u32 or LookaheadType == u64);51 assert(Lookahead == u32 or Lookahead == u64);
53 const BitReaderType = BitReader(LookaheadType);52 const LookaheadBitReader = BitReader(Lookahead);
5453
55 return struct {54 return struct {
56 const F = BitReaderType.flag;55 bits: LookaheadBitReader,
57
58 bits: BitReaderType,
59 hist: CircularBuffer = .{},56 hist: CircularBuffer = .{},
60 // Hashes, produces checkusm, of uncompressed data for gzip/zlib footer.57 // Hashes, produces checksum, of uncompressed data for gzip/zlib footer.
61 hasher: container.Hasher() = .{},58 hasher: container.Hasher() = .{},
6259
63 // dynamic block huffman code decoders60 // dynamic block huffman code decoders
...@@ -79,7 +76,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -79,7 +76,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
7976
80 const Self = @This();77 const Self = @This();
8178
82 pub const Error = BitReaderType.Error || Container.Error || hfd.Error || error{79 pub const Error = anyerror || Container.Error || hfd.Error || error{
83 InvalidCode,80 InvalidCode,
84 InvalidMatch,81 InvalidMatch,
85 InvalidBlockType,82 InvalidBlockType,
...@@ -88,10 +85,10 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -88,10 +85,10 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
88 };85 };
8986
90 pub fn init(bw: *std.io.BufferedReader) Self {87 pub fn init(bw: *std.io.BufferedReader) Self {
91 return .{ .bits = BitReaderType.init(bw) };88 return .{ .bits = LookaheadBitReader.init(bw) };
92 }89 }
9390
94 fn blockHeader(self: *Self) !void {91 fn blockHeader(self: *Self) anyerror!void {
95 self.bfinal = try self.bits.read(u1);92 self.bfinal = try self.bits.read(u1);
96 self.block_type = try self.bits.read(u2);93 self.block_type = try self.bits.read(u2);
97 }94 }
...@@ -129,7 +126,10 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -129,7 +126,10 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
129 fn fixedDistanceCode(self: *Self, code: u8) !void {126 fn fixedDistanceCode(self: *Self, code: u8) !void {
130 try self.bits.fill(5 + 5 + 13);127 try self.bits.fill(5 + 5 + 13);
131 const length = try self.decodeLength(code);128 const length = try self.decodeLength(code);
132 const distance = try self.decodeDistance(try self.bits.readF(u5, F.buffered | F.reverse));129 const distance = try self.decodeDistance(try self.bits.readF(u5, .{
130 .buffered = true,
131 .reverse = true,
132 }));
133 try self.hist.writeMatch(length, distance);133 try self.hist.writeMatch(length, distance);
134 }134 }
135135
...@@ -139,7 +139,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -139,7 +139,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
139 return if (ml.extra_bits == 0) // 0 - 5 extra bits139 return if (ml.extra_bits == 0) // 0 - 5 extra bits
140 ml.base140 ml.base
141 else141 else
142 ml.base + try self.bits.readN(ml.extra_bits, F.buffered);142 ml.base + try self.bits.readN(ml.extra_bits, .{ .buffered = true });
143 }143 }
144144
145 fn decodeDistance(self: *Self, code: u8) !u16 {145 fn decodeDistance(self: *Self, code: u8) !u16 {
...@@ -148,7 +148,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -148,7 +148,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
148 return if (md.extra_bits == 0) // 0 - 13 extra bits148 return if (md.extra_bits == 0) // 0 - 13 extra bits
149 md.base149 md.base
150 else150 else
151 md.base + try self.bits.readN(md.extra_bits, F.buffered);151 md.base + try self.bits.readN(md.extra_bits, .{ .buffered = true });
152 }152 }
153153
154 fn dynamicBlockHeader(self: *Self) !void {154 fn dynamicBlockHeader(self: *Self) !void {
...@@ -171,7 +171,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -171,7 +171,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
171 var dec_lens = [_]u4{0} ** (286 + 30);171 var dec_lens = [_]u4{0} ** (286 + 30);
172 var pos: usize = 0;172 var pos: usize = 0;
173 while (pos < hlit + hdist) {173 while (pos < hlit + hdist) {
174 const sym = try cl_dec.find(try self.bits.peekF(u7, F.reverse));174 const sym = try cl_dec.find(try self.bits.peekF(u7, .{ .reverse = true }));
175 try self.bits.shift(sym.code_bits);175 try self.bits.shift(sym.code_bits);
176 pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);176 pos += try self.dynamicCodeLength(sym.symbol, &dec_lens, pos);
177 }177 }
...@@ -230,13 +230,13 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -230,13 +230,13 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
230 .literal => self.hist.write(sym.symbol),230 .literal => self.hist.write(sym.symbol),
231 .match => { // Decode match backreference <length, distance>231 .match => { // Decode match backreference <length, distance>
232 // fill so we can use buffered reads232 // fill so we can use buffered reads
233 if (LookaheadType == u32)233 if (Lookahead == u32)
234 try self.bits.fill(5 + 15)234 try self.bits.fill(5 + 15)
235 else235 else
236 try self.bits.fill(5 + 15 + 13);236 try self.bits.fill(5 + 15 + 13);
237 const length = try self.decodeLength(sym.symbol);237 const length = try self.decodeLength(sym.symbol);
238 const dsm = try self.decodeSymbol(&self.dst_dec);238 const dsm = try self.decodeSymbol(&self.dst_dec);
239 if (LookaheadType == u32) try self.bits.fill(13);239 if (Lookahead == u32) try self.bits.fill(13);
240 const distance = try self.decodeDistance(dsm.symbol);240 const distance = try self.decodeDistance(dsm.symbol);
241 try self.hist.writeMatch(length, distance);241 try self.hist.writeMatch(length, distance);
242 },242 },
...@@ -251,7 +251,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -251,7 +251,7 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
251 // used. Shift bit reader for that much bits, those bits are used. And251 // used. Shift bit reader for that much bits, those bits are used. And
252 // return symbol.252 // return symbol.
253 fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {253 fn decodeSymbol(self: *Self, decoder: anytype) !hfd.Symbol {
254 const sym = try decoder.find(try self.bits.peekF(u15, F.buffered | F.reverse));254 const sym = try decoder.find(try self.bits.peekF(u15, .{ .buffered = true, .reverse = true }));
255 try self.bits.shift(sym.code_bits);255 try self.bits.shift(sym.code_bits);
256 return sym;256 return sym;
257 }257 }
...@@ -338,22 +338,48 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type...@@ -338,22 +338,48 @@ pub fn Inflate(comptime container: Container, comptime LookaheadType: type) type
338 }338 }
339 }339 }
340340
341 // Reader interface341 fn reader_streamRead(
342 ctx: ?*anyopaque,
343 bw: *std.io.BufferedWriter,
344 limit: std.io.Reader.Limit,
345 ) std.io.Reader.RwResult {
346 const self: *Self = @alignCast(@ptrCast(ctx));
347 const out = bw.writableSlice(1) catch |err| return .{ .write_err = err };
348 const in = self.get(limit.min(out.len)) catch |err| return .{ .read_err = err };
349 if (in.len == 0) return .{ .read_end = true };
350 @memcpy(out[0..in.len], in);
351 return .{ .len = in.len };
352 }
342353
343 pub const Reader = std.io.Reader(*Self, Error, read);354 fn reader_streamReadVec(ctx: ?*anyopaque, data: []const []u8) std.io.Reader.Result {
355 const self: *Self = @alignCast(@ptrCast(ctx));
356 var total: usize = 0;
357 for (data) |buffer| {
358 if (buffer.len == 0) break;
359 const out = self.get(buffer.len) catch |err| {
360 return .{ .len = total, .err = err };
361 };
362 if (out.len == 0) break;
363 @memcpy(buffer[0..out.len], out);
364 total += out.len;
365 }
366 return .{ .len = total, .end = total == 0 };
367 }
344368
345 /// Returns the number of bytes read. It may be less than buffer.len.369 pub fn streamReadVec(self: *Self, data: []const []u8) std.io.Reader.Result {
346 /// If the number of bytes read is 0, it means end of stream.370 return reader_streamReadVec(self, data);
347 /// End of stream is not an error condition.
348 pub fn read(self: *Self, buffer: []u8) Error!usize {
349 if (buffer.len == 0) return 0;
350 const out = try self.get(buffer.len);
351 @memcpy(buffer[0..out.len], out);
352 return out.len;
353 }371 }
354372
355 pub fn reader(self: *Self) Reader {373 pub fn reader(self: *Self) std.io.Reader {
356 return .{ .context = self };374 return .{
375 .context = self,
376 .vtable = &.{
377 .posRead = null,
378 .posReadVec = null,
379 .streamRead = reader_streamRead,
380 .streamReadVec = reader_streamReadVec,
381 },
382 };
357 }383 }
358 };384 };
359}385}
...@@ -567,3 +593,427 @@ test "bug 19895" {...@@ -567,3 +593,427 @@ test "bug 19895" {
567 var buf: [0]u8 = undefined;593 var buf: [0]u8 = undefined;
568 try testing.expectEqual(0, try decomp.read(&buf));594 try testing.expectEqual(0, try decomp.read(&buf));
569}595}
596
597/// Bit reader used during inflate (decompression). Has internal buffer of 64
598/// bits which shifts right after bits are consumed. Uses forward_reader to fill
599/// that internal buffer when needed.
600///
601/// readF is the core function. Supports few different ways of getting bits
602/// controlled by flags. In hot path we try to avoid checking whether we need to
603/// fill buffer from forward_reader by calling fill in advance and readF with
604/// buffered flag set.
605///
606pub fn BitReader(comptime T: type) type {
607 assert(T == u32 or T == u64);
608 const t_bytes: usize = @sizeOf(T);
609 const Tshift = if (T == u64) u6 else u5;
610
611 return struct {
612 // Underlying reader used for filling internal bits buffer
613 forward_reader: *std.io.BufferedReader,
614 // Internal buffer of 64 bits
615 bits: T = 0,
616 // Number of bits in the buffer
617 nbits: u32 = 0,
618
619 const Self = @This();
620
621 pub const Flags = packed struct(u3) {
622 /// dont advance internal buffer, just get bits, leave them in buffer
623 peek: bool = false,
624 /// assume that there is no need to fill, fill should be called before
625 buffered: bool = false,
626 /// bit reverse read bits
627 reverse: bool = false,
628
629 /// work around https://github.com/ziglang/zig/issues/18882
630 pub inline fn toInt(f: Flags) u3 {
631 return @bitCast(f);
632 }
633 };
634
635 pub fn init(forward_reader: *std.io.BufferedReader) Self {
636 var self = Self{ .forward_reader = forward_reader };
637 self.fill(1) catch {};
638 return self;
639 }
640
641 /// Try to have `nice` bits are available in buffer. Reads from
642 /// forward reader if there is no `nice` bits in buffer. Returns error
643 /// if end of forward stream is reached and internal buffer is empty.
644 /// It will not error if less than `nice` bits are in buffer, only when
645 /// all bits are exhausted. During inflate we usually know what is the
646 /// maximum bits for the next step but usually that step will need less
647 /// bits to decode. So `nice` is not hard limit, it will just try to have
648 /// that number of bits available. If end of forward stream is reached
649 /// it may be some extra zero bits in buffer.
650 pub fn fill(self: *Self, nice: u6) !void {
651 if (self.nbits >= nice and nice != 0) {
652 return; // We have enough bits
653 }
654 // Read more bits from forward reader
655
656 // Number of empty bytes in bits, round nbits to whole bytes.
657 const empty_bytes =
658 @as(u8, if (self.nbits & 0x7 == 0) t_bytes else t_bytes - 1) - // 8 for 8, 16, 24..., 7 otherwise
659 (self.nbits >> 3); // 0 for 0-7, 1 for 8-16, ... same as / 8
660
661 var buf: [t_bytes]u8 = [_]u8{0} ** t_bytes;
662 const bytes_read = self.forward_reader.partialRead(buf[0..empty_bytes]) catch 0;
663 if (bytes_read > 0) {
664 const u: T = std.mem.readInt(T, buf[0..t_bytes], .little);
665 self.bits |= u << @as(Tshift, @intCast(self.nbits));
666 self.nbits += 8 * @as(u8, @intCast(bytes_read));
667 return;
668 }
669
670 if (self.nbits == 0)
671 return error.EndOfStream;
672 }
673
674 /// Read exactly buf.len bytes into buf.
675 pub fn readAll(self: *Self, buf: []u8) anyerror!void {
676 assert(self.alignBits() == 0); // internal bits must be at byte boundary
677
678 // First read from internal bits buffer.
679 var n: usize = 0;
680 while (self.nbits > 0 and n < buf.len) {
681 buf[n] = try self.readF(u8, .{ .buffered = true });
682 n += 1;
683 }
684 // Then use forward reader for all other bytes.
685 try self.forward_reader.read(buf[n..]);
686 }
687
688 /// Alias for readF(U, 0).
689 pub fn read(self: *Self, comptime U: type) !U {
690 return self.readF(U, .{});
691 }
692
693 /// Alias for readF with flag.peak set.
694 pub inline fn peekF(self: *Self, comptime U: type, comptime how: Flags) !U {
695 return self.readF(U, .{
696 .peek = true,
697 .buffered = how.buffered,
698 .reverse = how.reverse,
699 });
700 }
701
702 /// Read with flags provided.
703 pub fn readF(self: *Self, comptime U: type, comptime how: Flags) !U {
704 if (U == T) {
705 assert(how.toInt() == 0);
706 assert(self.alignBits() == 0);
707 try self.fill(@bitSizeOf(T));
708 if (self.nbits != @bitSizeOf(T)) return error.EndOfStream;
709 const v = self.bits;
710 self.nbits = 0;
711 self.bits = 0;
712 return v;
713 }
714 const n: Tshift = @bitSizeOf(U);
715 // work around https://github.com/ziglang/zig/issues/18882
716 switch (how.toInt()) {
717 @as(Flags, .{}).toInt() => { // `normal` read
718 try self.fill(n); // ensure that there are n bits in the buffer
719 const u: U = @truncate(self.bits); // get n bits
720 try self.shift(n); // advance buffer for n
721 return u;
722 },
723 @as(Flags, .{ .peek = true }).toInt() => { // no shift, leave bits in the buffer
724 try self.fill(n);
725 return @truncate(self.bits);
726 },
727 @as(Flags, .{ .buffered = true }).toInt() => { // no fill, assume that buffer has enough bits
728 const u: U = @truncate(self.bits);
729 try self.shift(n);
730 return u;
731 },
732 @as(Flags, .{ .reverse = true }).toInt() => { // same as 0 with bit reverse
733 try self.fill(n);
734 const u: U = @truncate(self.bits);
735 try self.shift(n);
736 return @bitReverse(u);
737 },
738 @as(Flags, .{ .peek = true, .reverse = true }).toInt() => {
739 try self.fill(n);
740 return @bitReverse(@as(U, @truncate(self.bits)));
741 },
742 @as(Flags, .{ .buffered = true, .reverse = true }).toInt() => {
743 const u: U = @truncate(self.bits);
744 try self.shift(n);
745 return @bitReverse(u);
746 },
747 @as(Flags, .{ .peek = true, .buffered = true }).toInt() => {
748 return @truncate(self.bits);
749 },
750 @as(Flags, .{ .peek = true, .buffered = true, .reverse = true }).toInt() => {
751 return @bitReverse(@as(U, @truncate(self.bits)));
752 },
753 }
754 }
755
756 /// Read n number of bits.
757 /// Only buffered flag can be used in how.
758 pub fn readN(self: *Self, n: u4, comptime how: Flags) !u16 {
759 // work around https://github.com/ziglang/zig/issues/18882
760 switch (how.toInt()) {
761 @as(Flags, .{}).toInt() => {
762 try self.fill(n);
763 },
764 @as(Flags, .{ .buffered = true }).toInt() => {},
765 else => unreachable,
766 }
767 const mask: u16 = (@as(u16, 1) << n) - 1;
768 const u: u16 = @as(u16, @truncate(self.bits)) & mask;
769 try self.shift(n);
770 return u;
771 }
772
773 /// Advance buffer for n bits.
774 pub fn shift(self: *Self, n: Tshift) !void {
775 if (n > self.nbits) return error.EndOfStream;
776 self.bits >>= n;
777 self.nbits -= n;
778 }
779
780 /// Skip n bytes.
781 pub fn skipBytes(self: *Self, n: u16) !void {
782 for (0..n) |_| {
783 try self.fill(8);
784 try self.shift(8);
785 }
786 }
787
788 // Number of bits to align stream to the byte boundary.
789 fn alignBits(self: *Self) u3 {
790 return @intCast(self.nbits & 0x7);
791 }
792
793 /// Align stream to the byte boundary.
794 pub fn alignToByte(self: *Self) void {
795 const ab = self.alignBits();
796 if (ab > 0) self.shift(ab) catch unreachable;
797 }
798
799 /// Skip zero terminated string.
800 pub fn skipStringZ(self: *Self) !void {
801 while (true) {
802 if (try self.readF(u8, 0) == 0) break;
803 }
804 }
805
806 /// Read deflate fixed fixed code.
807 /// Reads first 7 bits, and then maybe 1 or 2 more to get full 7,8 or 9 bit code.
808 /// ref: https://datatracker.ietf.org/doc/html/rfc1951#page-12
809 /// Lit Value Bits Codes
810 /// --------- ---- -----
811 /// 0 - 143 8 00110000 through
812 /// 10111111
813 /// 144 - 255 9 110010000 through
814 /// 111111111
815 /// 256 - 279 7 0000000 through
816 /// 0010111
817 /// 280 - 287 8 11000000 through
818 /// 11000111
819 pub fn readFixedCode(self: *Self) !u16 {
820 try self.fill(7 + 2);
821 const code7 = try self.readF(u7, .{ .buffered = true, .reverse = true });
822 if (code7 <= 0b0010_111) { // 7 bits, 256-279, codes 0000_000 - 0010_111
823 return @as(u16, code7) + 256;
824 } else if (code7 <= 0b1011_111) { // 8 bits, 0-143, codes 0011_0000 through 1011_1111
825 return (@as(u16, code7) << 1) + @as(u16, try self.readF(u1, .{ .buffered = true })) - 0b0011_0000;
826 } else if (code7 <= 0b1100_011) { // 8 bit, 280-287, codes 1100_0000 - 1100_0111
827 return (@as(u16, code7 - 0b1100000) << 1) + try self.readF(u1, .{ .buffered = true }) + 280;
828 } else { // 9 bit, 144-255, codes 1_1001_0000 - 1_1111_1111
829 return (@as(u16, code7 - 0b1100_100) << 2) + @as(u16, try self.readF(u2, .{ .buffered = true, .reverse = true })) + 144;
830 }
831 }
832 };
833}
834
835test "readF" {
836 var input: std.io.BufferedReader = undefined;
837 input.initFixed(&[_]u8{ 0xf3, 0x48, 0xcd, 0xc9, 0x00, 0x00 });
838 var br: BitReader(u64) = .init(&input);
839
840 try testing.expectEqual(@as(u8, 48), br.nbits);
841 try testing.expectEqual(@as(u64, 0xc9cd48f3), br.bits);
842
843 try testing.expect(try br.readF(u1, 0) == 0b0000_0001);
844 try testing.expect(try br.readF(u2, 0) == 0b0000_0001);
845 try testing.expectEqual(@as(u8, 48 - 3), br.nbits);
846 try testing.expectEqual(@as(u3, 5), br.alignBits());
847
848 try testing.expect(try br.readF(u8, .{ .peek = true }) == 0b0001_1110);
849 try testing.expect(try br.readF(u9, .{ .peek = true }) == 0b1_0001_1110);
850 try br.shift(9);
851 try testing.expectEqual(@as(u8, 36), br.nbits);
852 try testing.expectEqual(@as(u3, 4), br.alignBits());
853
854 try testing.expect(try br.readF(u4, 0) == 0b0100);
855 try testing.expectEqual(@as(u8, 32), br.nbits);
856 try testing.expectEqual(@as(u3, 0), br.alignBits());
857
858 try br.shift(1);
859 try testing.expectEqual(@as(u3, 7), br.alignBits());
860 try br.shift(1);
861 try testing.expectEqual(@as(u3, 6), br.alignBits());
862 br.alignToByte();
863 try testing.expectEqual(@as(u3, 0), br.alignBits());
864
865 try testing.expectEqual(@as(u64, 0xc9), br.bits);
866 try testing.expectEqual(@as(u16, 0x9), try br.readN(4, 0));
867 try testing.expectEqual(@as(u16, 0xc), try br.readN(4, 0));
868}
869
870test "read block type 1 data" {
871 inline for ([_]type{ u64, u32 }) |T| {
872 const data = [_]u8{
873 0xf3, 0x48, 0xcd, 0xc9, 0xc9, 0x57, 0x28, 0xcf, // deflate data block type 1
874 0x2f, 0xca, 0x49, 0xe1, 0x02, 0x00,
875 0x0c, 0x01, 0x02, 0x03, //
876 0xaa, 0xbb, 0xcc, 0xdd,
877 };
878 var fbs: std.io.BufferedReader = undefined;
879 fbs.initFixed(&data);
880 var br: BitReader(T) = .init(&fbs);
881
882 try testing.expectEqual(@as(u1, 1), try br.readF(u1, 0)); // bfinal
883 try testing.expectEqual(@as(u2, 1), try br.readF(u2, 0)); // block_type
884
885 for ("Hello world\n") |c| {
886 try testing.expectEqual(@as(u8, c), try br.readF(u8, .{ .reverse = true }) - 0x30);
887 }
888 try testing.expectEqual(@as(u7, 0), try br.readF(u7, 0)); // end of block
889 br.alignToByte();
890 try testing.expectEqual(@as(u32, 0x0302010c), try br.readF(u32, 0));
891 try testing.expectEqual(@as(u16, 0xbbaa), try br.readF(u16, 0));
892 try testing.expectEqual(@as(u16, 0xddcc), try br.readF(u16, 0));
893 }
894}
895
896test "shift/fill" {
897 const data = [_]u8{
898 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
899 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
900 };
901 var fbs: std.io.BufferedReader = undefined;
902 fbs.initFixed(&data);
903 var br: BitReader(u64) = .init(&fbs);
904
905 try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits);
906 try br.shift(8);
907 try testing.expectEqual(@as(u64, 0x00_08_07_06_05_04_03_02), br.bits);
908 try br.fill(60); // fill with 1 byte
909 try testing.expectEqual(@as(u64, 0x01_08_07_06_05_04_03_02), br.bits);
910 try br.shift(8 * 4 + 4);
911 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_10_80_70), br.bits);
912
913 try br.fill(60); // fill with 4 bytes (shift by 4)
914 try testing.expectEqual(@as(u64, 0x00_50_40_30_20_10_80_70), br.bits);
915 try testing.expectEqual(@as(u8, 8 * 7 + 4), br.nbits);
916
917 try br.shift(@intCast(br.nbits)); // clear buffer
918 try br.fill(8); // refill with the rest of the bytes
919 try testing.expectEqual(@as(u64, 0x00_00_00_00_00_08_07_06), br.bits);
920}
921
922test "readAll" {
923 inline for ([_]type{ u64, u32 }) |T| {
924 const data = [_]u8{
925 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
926 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08,
927 };
928 var fbs: std.io.BufferedReader = undefined;
929 fbs.initFixed(&data);
930 var br: BitReader(T) = .init(&fbs);
931
932 switch (T) {
933 u64 => try testing.expectEqual(@as(u64, 0x08_07_06_05_04_03_02_01), br.bits),
934 u32 => try testing.expectEqual(@as(u32, 0x04_03_02_01), br.bits),
935 else => unreachable,
936 }
937
938 var out: [16]u8 = undefined;
939 try br.readAll(out[0..]);
940 try testing.expect(br.nbits == 0);
941 try testing.expect(br.bits == 0);
942
943 try testing.expectEqualSlices(u8, data[0..16], &out);
944 }
945}
946
947test "readFixedCode" {
948 inline for ([_]type{ u64, u32 }) |T| {
949 const fixed_codes = @import("huffman_encoder.zig").fixed_codes;
950
951 var fbs: std.io.BufferedReader = undefined;
952 fbs.initFixed(&fixed_codes);
953 var rdr: BitReader(T) = .init(&fbs);
954
955 for (0..286) |c| {
956 try testing.expectEqual(c, try rdr.readFixedCode());
957 }
958 try testing.expect(rdr.nbits == 0);
959 }
960}
961
962test "u32 leaves no bits on u32 reads" {
963 const data = [_]u8{
964 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
965 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
966 };
967 var fbs: std.io.BufferedReader = undefined;
968 fbs.initFixed(&data);
969 var br: BitReader(u32) = .init(&fbs);
970
971 _ = try br.read(u3);
972 try testing.expectEqual(29, br.nbits);
973 br.alignToByte();
974 try testing.expectEqual(24, br.nbits);
975 try testing.expectEqual(0x04_03_02_01, try br.read(u32));
976 try testing.expectEqual(0, br.nbits);
977 try testing.expectEqual(0x08_07_06_05, try br.read(u32));
978 try testing.expectEqual(0, br.nbits);
979
980 _ = try br.read(u9);
981 try testing.expectEqual(23, br.nbits);
982 br.alignToByte();
983 try testing.expectEqual(16, br.nbits);
984 try testing.expectEqual(0x0e_0d_0c_0b, try br.read(u32));
985 try testing.expectEqual(0, br.nbits);
986}
987
988test "u64 need fill after alignToByte" {
989 const data = [_]u8{
990 0xff, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
991 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f,
992 };
993
994 // without fill
995 var fbs: std.io.BufferedReader = undefined;
996 fbs.initFixed(&data);
997 var br: BitReader(u64) = .init(&fbs);
998 _ = try br.read(u23);
999 try testing.expectEqual(41, br.nbits);
1000 br.alignToByte();
1001 try testing.expectEqual(40, br.nbits);
1002 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
1003 try testing.expectEqual(8, br.nbits);
1004 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
1005 try testing.expectEqual(32, br.nbits);
1006
1007 // fill after align ensures all bits filled
1008 fbs.reset();
1009 br = .init(&fbs);
1010 _ = try br.read(u23);
1011 try testing.expectEqual(41, br.nbits);
1012 br.alignToByte();
1013 try br.fill(0);
1014 try testing.expectEqual(64, br.nbits);
1015 try testing.expectEqual(0x06_05_04_03, try br.read(u32));
1016 try testing.expectEqual(32, br.nbits);
1017 try testing.expectEqual(0x0a_09_08_07, try br.read(u32));
1018 try testing.expectEqual(0, br.nbits);
1019}
lib/std/compress/lzma.zig+906-60
...@@ -1,90 +1,936 @@...@@ -1,90 +1,936 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const assert = std.debug.assert;
2const math = std.math;3const math = std.math;
3const mem = std.mem;4const mem = std.mem;
4const Allocator = std.mem.Allocator;5const Allocator = std.mem.Allocator;
6const testing = std.testing;
7const expectEqualSlices = std.testing.expectEqualSlices;
8const expectError = std.testing.expectError;
59
6pub const decode = @import("lzma/decode.zig");10pub const RangeDecoder = struct {
11 range: u32,
12 code: u32,
713
8pub fn decompress(14 pub fn init(rd: *RangeDecoder, br: *std.io.BufferedReader) anyerror!usize {
9 allocator: Allocator,15 const reserved = try br.takeByte();
10 reader: anytype,16 if (reserved != 0) return error.CorruptInput;
11) !Decompress(@TypeOf(reader)) {17 rd.* = .{
12 return decompressWithOptions(allocator, reader, .{});18 .range = 0xFFFF_FFFF,
13}19 .code = try br.takeInt(u32, .big),
20 };
21 return 5;
22 }
1423
15pub fn decompressWithOptions(24 pub inline fn isFinished(self: RangeDecoder) bool {
16 allocator: Allocator,25 return self.code == 0;
17 reader: anytype,26 }
18 options: decode.Options,27
19) !Decompress(@TypeOf(reader)) {28 inline fn normalize(self: *RangeDecoder, br: *std.io.BufferedReader) !void {
20 const params = try decode.Params.readHeader(reader, options);29 if (self.range < 0x0100_0000) {
21 return Decompress(@TypeOf(reader)).init(allocator, reader, params, options.memlimit);30 self.range <<= 8;
22}31 self.code = (self.code << 8) ^ @as(u32, try br.takeByte());
32 }
33 }
34
35 inline fn getBit(self: *RangeDecoder, br: *std.io.BufferedReader) !bool {
36 self.range >>= 1;
37
38 const bit = self.code >= self.range;
39 if (bit)
40 self.code -= self.range;
41
42 try self.normalize(br);
43 return bit;
44 }
45
46 pub fn get(self: *RangeDecoder, br: *std.io.BufferedReader, count: usize) !u32 {
47 var result: u32 = 0;
48 var i: usize = 0;
49 while (i < count) : (i += 1)
50 result = (result << 1) ^ @intFromBool(try self.getBit(br));
51 return result;
52 }
53
54 pub inline fn decodeBit(self: *RangeDecoder, br: *std.io.BufferedReader, prob: *u16, update: bool) !bool {
55 const bound = (self.range >> 11) * prob.*;
56
57 if (self.code < bound) {
58 if (update)
59 prob.* += (0x800 - prob.*) >> 5;
60 self.range = bound;
61
62 try self.normalize(br);
63 return false;
64 } else {
65 if (update)
66 prob.* -= prob.* >> 5;
67 self.code -= bound;
68 self.range -= bound;
2369
24pub fn Decompress(comptime ReaderType: type) type {70 try self.normalize(br);
71 return true;
72 }
73 }
74
75 fn parseBitTree(
76 self: *RangeDecoder,
77 br: *std.io.BufferedReader,
78 num_bits: u5,
79 probs: []u16,
80 update: bool,
81 ) !u32 {
82 var tmp: u32 = 1;
83 var i: @TypeOf(num_bits) = 0;
84 while (i < num_bits) : (i += 1) {
85 const bit = try self.decodeBit(br, &probs[tmp], update);
86 tmp = (tmp << 1) ^ @intFromBool(bit);
87 }
88 return tmp - (@as(u32, 1) << num_bits);
89 }
90
91 pub fn parseReverseBitTree(
92 self: *RangeDecoder,
93 br: *std.io.BufferedReader,
94 num_bits: u5,
95 probs: []u16,
96 offset: usize,
97 update: bool,
98 ) !u32 {
99 var result: u32 = 0;
100 var tmp: usize = 1;
101 var i: @TypeOf(num_bits) = 0;
102 while (i < num_bits) : (i += 1) {
103 const bit = @intFromBool(try self.decodeBit(br, &probs[offset + tmp], update));
104 tmp = (tmp << 1) ^ bit;
105 result ^= @as(u32, bit) << i;
106 }
107 return result;
108 }
109};
110
111pub const LenDecoder = struct {
112 choice: u16 = 0x400,
113 choice2: u16 = 0x400,
114 low_coder: [16]BitTree(3) = @splat(.{}),
115 mid_coder: [16]BitTree(3) = @splat(.{}),
116 high_coder: BitTree(8) = .{},
117
118 pub fn decode(
119 self: *LenDecoder,
120 br: *std.io.BufferedReader,
121 decoder: *RangeDecoder,
122 pos_state: usize,
123 update: bool,
124 ) !usize {
125 if (!try decoder.decodeBit(br, &self.choice, update)) {
126 return @as(usize, try self.low_coder[pos_state].parse(br, decoder, update));
127 } else if (!try decoder.decodeBit(br, &self.choice2, update)) {
128 return @as(usize, try self.mid_coder[pos_state].parse(br, decoder, update)) + 8;
129 } else {
130 return @as(usize, try self.high_coder.parse(br, decoder, update)) + 16;
131 }
132 }
133
134 pub fn reset(self: *LenDecoder) void {
135 self.choice = 0x400;
136 self.choice2 = 0x400;
137 for (&self.low_coder) |*t| t.reset();
138 for (&self.mid_coder) |*t| t.reset();
139 self.high_coder.reset();
140 }
141};
142
143pub fn BitTree(comptime num_bits: usize) type {
25 return struct {144 return struct {
145 probs: [1 << num_bits]u16 = @splat(0x400),
146
26 const Self = @This();147 const Self = @This();
27148
28 pub const Error =149 pub fn parse(
29 ReaderType.Error ||150 self: *Self,
30 Allocator.Error ||151 br: *std.io.BufferedReader,
31 error{ CorruptInput, EndOfStream, Overflow };152 decoder: *RangeDecoder,
153 update: bool,
154 ) !u32 {
155 return decoder.parseBitTree(br, num_bits, &self.probs, update);
156 }
157
158 pub fn parseReverse(
159 self: *Self,
160 br: *std.io.BufferedReader,
161 decoder: *RangeDecoder,
162 update: bool,
163 ) !u32 {
164 return decoder.parseReverseBitTree(br, num_bits, &self.probs, 0, update);
165 }
32166
33 pub const Reader = std.io.Reader(*Self, Error, read);167 pub fn reset(self: *Self) void {
168 @memset(&self.probs, 0x400);
169 }
170 };
171}
34172
35 allocator: Allocator,173pub const Decode = struct {
36 in_reader: ReaderType,174 properties: Properties,
37 to_read: std.ArrayListUnmanaged(u8),175 unpacked_size: ?u64,
176 literal_probs: Vec2D(u16),
177 pos_slot_decoder: [4]BitTree(6),
178 align_decoder: BitTree(4),
179 pos_decoders: [115]u16,
180 is_match: [192]u16,
181 is_rep: [12]u16,
182 is_rep_g0: [12]u16,
183 is_rep_g1: [12]u16,
184 is_rep_g2: [12]u16,
185 is_rep_0long: [192]u16,
186 state: usize,
187 rep: [4]usize,
188 len_decoder: LenDecoder,
189 rep_len_decoder: LenDecoder,
38190
39 buffer: decode.lzbuffer.LzCircularBuffer,191 pub const Options = struct {
40 decoder: decode.rangecoder.RangeDecoder,192 unpacked_size: UnpackedSize = .read_from_header,
41 state: decode.DecoderState,193 memlimit: ?usize = null,
194 allow_incomplete: bool = false,
195 };
42196
43 pub fn init(allocator: Allocator, source: ReaderType, params: decode.Params, memlimit: ?usize) !Self {197 pub const UnpackedSize = union(enum) {
44 return Self{198 read_from_header,
45 .allocator = allocator,199 read_header_but_use_provided: ?u64,
46 .in_reader = source,200 use_provided: ?u64,
47 .to_read = .{},201 };
202
203 const ProcessingStatus = enum {
204 cont,
205 finished,
206 };
207
208 pub const Properties = struct {
209 lc: u4,
210 lp: u3,
211 pb: u3,
212
213 fn validate(self: Properties) void {
214 assert(self.lc <= 8);
215 assert(self.lp <= 4);
216 assert(self.pb <= 4);
217 }
218 };
219
220 pub const Params = struct {
221 properties: Properties,
222 dict_size: u32,
223 unpacked_size: ?u64,
224
225 pub fn readHeader(br: *std.io.BufferedReader, options: Options) anyerror!Params {
226 var props = try br.readByte();
227 if (props >= 225) {
228 return error.CorruptInput;
229 }
48230
49 .buffer = decode.lzbuffer.LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),231 const lc = @as(u4, @intCast(props % 9));
50 .decoder = try decode.rangecoder.RangeDecoder.init(source),232 props /= 9;
51 .state = try decode.DecoderState.init(allocator, params.properties, params.unpacked_size),233 const lp = @as(u3, @intCast(props % 5));
234 props /= 5;
235 const pb = @as(u3, @intCast(props));
236
237 const dict_size_provided = try br.readInt(u32, .little);
238 const dict_size = @max(0x1000, dict_size_provided);
239
240 const unpacked_size = switch (options.unpacked_size) {
241 .read_from_header => blk: {
242 const unpacked_size_provided = try br.readInt(u64, .little);
243 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
244 break :blk if (marker_mandatory)
245 null
246 else
247 unpacked_size_provided;
248 },
249 .read_header_but_use_provided => |x| blk: {
250 _ = try br.readInt(u64, .little);
251 break :blk x;
252 },
253 .use_provided => |x| x,
254 };
255
256 return Params{
257 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
258 .dict_size = dict_size,
259 .unpacked_size = unpacked_size,
52 };260 };
53 }261 }
262 };
263
264 pub fn init(
265 allocator: Allocator,
266 properties: Properties,
267 unpacked_size: ?u64,
268 ) !Decode {
269 return .{
270 .properties = properties,
271 .unpacked_size = unpacked_size,
272 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (properties.lc + properties.lp), 0x300 }),
273 .pos_slot_decoder = @splat(.{}),
274 .align_decoder = .{},
275 .pos_decoders = @splat(0x400),
276 .is_match = @splat(0x400),
277 .is_rep = @splat(0x400),
278 .is_rep_g0 = @splat(0x400),
279 .is_rep_g1 = @splat(0x400),
280 .is_rep_g2 = @splat(0x400),
281 .is_rep_0long = @splat(0x400),
282 .state = 0,
283 .rep = @splat(0),
284 .len_decoder = .{},
285 .rep_len_decoder = .{},
286 };
287 }
288
289 pub fn deinit(self: *Decode, allocator: Allocator) void {
290 self.literal_probs.deinit(allocator);
291 self.* = undefined;
292 }
54293
55 pub fn reader(self: *Self) Reader {294 pub fn resetState(self: *Decode, allocator: Allocator, new_props: Properties) !void {
56 return .{ .context = self };295 new_props.validate();
296 if (self.properties.lc + self.properties.lp == new_props.lc + new_props.lp) {
297 self.literal_probs.fill(0x400);
298 } else {
299 self.literal_probs.deinit(allocator);
300 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
57 }301 }
58302
59 pub fn deinit(self: *Self) void {303 self.properties = new_props;
60 self.to_read.deinit(self.allocator);304 for (&self.pos_slot_decoder) |*t| t.reset();
61 self.buffer.deinit(self.allocator);305 self.align_decoder.reset();
62 self.state.deinit(self.allocator);306 self.pos_decoders = @splat(0x400);
63 self.* = undefined;307 self.is_match = @splat(0x400);
308 self.is_rep = @splat(0x400);
309 self.is_rep_g0 = @splat(0x400);
310 self.is_rep_g1 = @splat(0x400);
311 self.is_rep_g2 = @splat(0x400);
312 self.is_rep_0long = @splat(0x400);
313 self.state = 0;
314 self.rep = @splat(0);
315 self.len_decoder.reset();
316 self.rep_len_decoder.reset();
317 }
318
319 fn processNextInner(
320 self: *Decode,
321 allocator: Allocator,
322 br: *std.io.BufferedReader,
323 bw: *std.io.BufferedWriter,
324 buffer: anytype,
325 decoder: *RangeDecoder,
326 bytes_read: *usize,
327 update: bool,
328 ) !ProcessingStatus {
329 const pos_state = buffer.len & ((@as(usize, 1) << self.properties.pb) - 1);
330
331 if (!try decoder.decodeBit(br, &self.is_match[(self.state << 4) + pos_state], update, bytes_read)) {
332 const byte: u8 = try self.decodeLiteral(br, buffer, decoder, update, bytes_read);
333
334 if (update) {
335 try buffer.appendLiteral(allocator, byte, bw);
336
337 self.state = if (self.state < 4)
338 0
339 else if (self.state < 10)
340 self.state - 3
341 else
342 self.state - 6;
343 }
344 return .cont;
345 }
346
347 var len: usize = undefined;
348 if (try decoder.decodeBit(br, &self.is_rep[self.state], update, bytes_read)) {
349 if (!try decoder.decodeBit(br, &self.is_rep_g0[self.state], update, bytes_read)) {
350 if (!try decoder.decodeBit(br, &self.is_rep_0long[(self.state << 4) + pos_state], update, bytes_read)) {
351 if (update) {
352 self.state = if (self.state < 7) 9 else 11;
353 const dist = self.rep[0] + 1;
354 try buffer.appendLz(allocator, 1, dist, bw);
355 }
356 return .cont;
357 }
358 } else {
359 const idx: usize = if (!try decoder.decodeBit(br, &self.is_rep_g1[self.state], update, bytes_read))
360 1
361 else if (!try decoder.decodeBit(br, &self.is_rep_g2[self.state], update, bytes_read))
362 2
363 else
364 3;
365 if (update) {
366 const dist = self.rep[idx];
367 var i = idx;
368 while (i > 0) : (i -= 1) {
369 self.rep[i] = self.rep[i - 1];
370 }
371 self.rep[0] = dist;
372 }
373 }
374
375 len = try self.rep_len_decoder.decode(br, decoder, pos_state, update, bytes_read);
376
377 if (update) {
378 self.state = if (self.state < 7) 8 else 11;
379 }
380 } else {
381 if (update) {
382 self.rep[3] = self.rep[2];
383 self.rep[2] = self.rep[1];
384 self.rep[1] = self.rep[0];
385 }
386
387 len = try self.len_decoder.decode(br, decoder, pos_state, update, bytes_read);
388
389 if (update) {
390 self.state = if (self.state < 7) 7 else 10;
391 }
392
393 const rep_0 = try self.decodeDistance(br, decoder, len, update, bytes_read);
394
395 if (update) {
396 self.rep[0] = rep_0;
397 if (self.rep[0] == 0xFFFF_FFFF) {
398 if (decoder.isFinished()) {
399 return .finished;
400 }
401 return error.CorruptInput;
402 }
403 }
404 }
405
406 if (update) {
407 len += 2;
408
409 const dist = self.rep[0] + 1;
410 try buffer.appendLz(allocator, len, dist, bw);
411 }
412
413 return .cont;
414 }
415
416 fn processNext(
417 self: *Decode,
418 allocator: Allocator,
419 br: *std.io.BufferedReader,
420 bw: *std.io.BufferedWriter,
421 buffer: anytype,
422 decoder: *RangeDecoder,
423 bytes_read: *usize,
424 ) !ProcessingStatus {
425 return self.processNextInner(allocator, br, bw, buffer, decoder, bytes_read, true);
426 }
427
428 pub fn process(
429 self: *Decode,
430 allocator: Allocator,
431 br: *std.io.BufferedReader,
432 bw: *std.io.BufferedWriter,
433 buffer: anytype,
434 decoder: *RangeDecoder,
435 bytes_read: *usize,
436 ) !ProcessingStatus {
437 process_next: {
438 if (self.unpacked_size) |unpacked_size| {
439 if (buffer.len >= unpacked_size) {
440 break :process_next;
441 }
442 } else if (decoder.isFinished()) {
443 break :process_next;
444 }
445
446 switch (try self.processNext(allocator, br, bw, buffer, decoder, bytes_read)) {
447 .cont => return .cont,
448 .finished => break :process_next,
449 }
450 }
451
452 if (self.unpacked_size) |unpacked_size| {
453 if (buffer.len != unpacked_size) {
454 return error.CorruptInput;
455 }
64 }456 }
65457
66 pub fn read(self: *Self, output: []u8) Error!usize {458 return .finished;
67 const writer = self.to_read.writer(self.allocator);459 }
68 while (self.to_read.items.len < output.len) {460
69 switch (try self.state.process(self.allocator, self.in_reader, writer, &self.buffer, &self.decoder)) {461 fn decodeLiteral(
70 .continue_ => {},462 self: *Decode,
71 .finished => {463 br: *std.io.BufferedReader,
72 try self.buffer.finish(writer);464 buffer: anytype,
73 break;465 decoder: *RangeDecoder,
74 },466 update: bool,
467 bytes_read: *usize,
468 ) !u8 {
469 const def_prev_byte = 0;
470 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
471
472 var result: usize = 1;
473 const lit_state = ((buffer.len & ((@as(usize, 1) << self.properties.lp) - 1)) << self.properties.lc) +
474 (prev_byte >> (8 - self.properties.lc));
475 const probs = try self.literal_probs.getMut(lit_state);
476
477 if (self.state >= 7) {
478 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
479
480 while (result < 0x100) {
481 const match_bit = (match_byte >> 7) & 1;
482 match_byte <<= 1;
483 const bit = @intFromBool(try decoder.decodeBit(
484 br,
485 &probs[((@as(usize, 1) + match_bit) << 8) + result],
486 update,
487 bytes_read,
488 ));
489 result = (result << 1) ^ bit;
490 if (match_bit != bit) {
491 break;
75 }492 }
76 }493 }
77 const input = self.to_read.items;
78 const n = @min(input.len, output.len);
79 @memcpy(output[0..n], input[0..n]);
80 std.mem.copyForwards(u8, input[0 .. input.len - n], input[n..]);
81 self.to_read.shrinkRetainingCapacity(input.len - n);
82 return n;
83 }494 }
495
496 while (result < 0x100) {
497 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(br, &probs[result], update, bytes_read));
498 }
499
500 return @as(u8, @truncate(result - 0x100));
501 }
502
503 fn decodeDistance(
504 self: *Decode,
505 br: *std.io.BufferedReader,
506 decoder: *RangeDecoder,
507 length: usize,
508 update: bool,
509 bytes_read: *usize,
510 ) !usize {
511 const len_state = if (length > 3) 3 else length;
512
513 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(br, decoder, update, bytes_read));
514 if (pos_slot < 4)
515 return pos_slot;
516
517 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
518 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
519
520 if (pos_slot < 14) {
521 result += try decoder.parseReverseBitTree(
522 br,
523 num_direct_bits,
524 &self.pos_decoders,
525 result - pos_slot,
526 update,
527 bytes_read,
528 );
529 } else {
530 result += @as(usize, try decoder.get(br, num_direct_bits - 4, bytes_read)) << 4;
531 result += try self.align_decoder.parseReverse(br, decoder, update, bytes_read);
532 }
533
534 return result;
535 }
536};
537
538pub const Decompress = struct {
539 pub const Error =
540 anyerror ||
541 Allocator.Error ||
542 error{ CorruptInput, EndOfStream, Overflow };
543
544 allocator: Allocator,
545 in_reader: *std.io.BufferedReader,
546 to_read: std.ArrayListUnmanaged(u8),
547
548 buffer: LzCircularBuffer,
549 decoder: RangeDecoder,
550 state: Decode,
551
552 pub fn initOptions(allocator: Allocator, br: *std.io.BufferedReader, options: Decode.Options) !Decompress {
553 const params = try Decode.Params.readHeader(br, options);
554 return init(allocator, br, params, options.memlimit);
555 }
556
557 pub fn init(allocator: Allocator, source: *std.io.BufferedReader, params: Decode.Params, memlimit: ?usize) !Decompress {
558 return .{
559 .allocator = allocator,
560 .in_reader = source,
561 .to_read = .{},
562
563 .buffer = LzCircularBuffer.init(params.dict_size, memlimit orelse math.maxInt(usize)),
564 .decoder = try RangeDecoder.init(source),
565 .state = try Decode.init(allocator, params.properties, params.unpacked_size),
566 };
567 }
568
569 pub fn reader(self: *Decompress) std.io.Reader {
570 return .{ .context = self };
571 }
572
573 pub fn deinit(self: *Decompress) void {
574 self.to_read.deinit(self.allocator);
575 self.buffer.deinit(self.allocator);
576 self.state.deinit(self.allocator);
577 self.* = undefined;
578 }
579
580 pub fn read(self: *Decompress, output: []u8) Error!usize {
581 const bw = self.to_read.writer(self.allocator);
582 while (self.to_read.items.len < output.len) {
583 switch (try self.state.process(self.allocator, self.in_reader, bw, &self.buffer, &self.decoder)) {
584 .cont => {},
585 .finished => {
586 try self.buffer.finish(bw);
587 break;
588 },
589 }
590 }
591 const input = self.to_read.items;
592 const n = @min(input.len, output.len);
593 @memcpy(output[0..n], input[0..n]);
594 std.mem.copyForwards(u8, input[0 .. input.len - n], input[n..]);
595 self.to_read.shrinkRetainingCapacity(input.len - n);
596 return n;
597 }
598};
599
600/// A circular buffer for LZ sequences
601const LzCircularBuffer = struct {
602 /// Circular buffer
603 buf: std.ArrayListUnmanaged(u8),
604
605 /// Length of the buffer
606 dict_size: usize,
607
608 /// Buffer memory limit
609 memlimit: usize,
610
611 /// Current position
612 cursor: usize,
613
614 /// Total number of bytes sent through the buffer
615 len: usize,
616
617 const Self = @This();
618
619 pub fn init(dict_size: usize, memlimit: usize) Self {
620 return Self{
621 .buf = .{},
622 .dict_size = dict_size,
623 .memlimit = memlimit,
624 .cursor = 0,
625 .len = 0,
626 };
627 }
628
629 pub fn get(self: Self, index: usize) u8 {
630 return if (0 <= index and index < self.buf.items.len)
631 self.buf.items[index]
632 else
633 0;
634 }
635
636 pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
637 if (index >= self.memlimit) {
638 return error.CorruptInput;
639 }
640 try self.buf.ensureTotalCapacity(allocator, index + 1);
641 while (self.buf.items.len < index) {
642 self.buf.appendAssumeCapacity(0);
643 }
644 self.buf.appendAssumeCapacity(value);
645 }
646
647 /// Retrieve the last byte or return a default
648 pub fn lastOr(self: Self, lit: u8) u8 {
649 return if (self.len == 0)
650 lit
651 else
652 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
653 }
654
655 /// Retrieve the n-th last byte
656 pub fn lastN(self: Self, dist: usize) !u8 {
657 if (dist > self.dict_size or dist > self.len) {
658 return error.CorruptInput;
659 }
660
661 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
662 return self.get(offset);
663 }
664
665 /// Append a literal
666 pub fn appendLiteral(
667 self: *Self,
668 allocator: Allocator,
669 lit: u8,
670 bw: *std.io.BufferedWriter,
671 ) anyerror!void {
672 try self.set(allocator, self.cursor, lit);
673 self.cursor += 1;
674 self.len += 1;
675
676 // Flush the circular buffer to the output
677 if (self.cursor == self.dict_size) {
678 try bw.writeAll(self.buf.items);
679 self.cursor = 0;
680 }
681 }
682
683 /// Fetch an LZ sequence (length, distance) from inside the buffer
684 pub fn appendLz(
685 self: *Self,
686 allocator: Allocator,
687 len: usize,
688 dist: usize,
689 bw: *std.io.BufferedWriter,
690 ) anyerror!void {
691 if (dist > self.dict_size or dist > self.len) {
692 return error.CorruptInput;
693 }
694
695 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
696 var i: usize = 0;
697 while (i < len) : (i += 1) {
698 const x = self.get(offset);
699 try self.appendLiteral(allocator, x, bw);
700 offset += 1;
701 if (offset == self.dict_size) {
702 offset = 0;
703 }
704 }
705 }
706
707 pub fn finish(self: *Self, bw: *std.io.BufferedWriter) anyerror!void {
708 if (self.cursor > 0) {
709 try bw.writeAll(self.buf.items[0..self.cursor]);
710 self.cursor = 0;
711 }
712 }
713
714 pub fn deinit(self: *Self, allocator: Allocator) void {
715 self.buf.deinit(allocator);
716 self.* = undefined;
717 }
718};
719
720pub fn Vec2D(comptime T: type) type {
721 return struct {
722 data: []T,
723 cols: usize,
724
725 const Self = @This();
726
727 pub fn init(allocator: Allocator, value: T, size: struct { usize, usize }) !Self {
728 const len = try math.mul(usize, size[0], size[1]);
729 const data = try allocator.alloc(T, len);
730 @memset(data, value);
731 return Self{
732 .data = data,
733 .cols = size[1],
734 };
735 }
736
737 pub fn deinit(self: *Self, allocator: Allocator) void {
738 allocator.free(self.data);
739 self.* = undefined;
740 }
741
742 pub fn fill(self: *Self, value: T) void {
743 @memset(self.data, value);
744 }
745
746 inline fn _get(self: Self, row: usize) ![]T {
747 const start_row = try math.mul(usize, row, self.cols);
748 const end_row = try math.add(usize, start_row, self.cols);
749 return self.data[start_row..end_row];
750 }
751
752 pub fn get(self: Self, row: usize) ![]const T {
753 return self._get(row);
754 }
755
756 pub fn getMut(self: *Self, row: usize) ![]T {
757 return self._get(row);
758 }
759 };
760}
761
762test "Vec2D init" {
763 const allocator = testing.allocator;
764 var vec2d = try Vec2D(i32).init(allocator, 1, .{ 2, 3 });
765 defer vec2d.deinit(allocator);
766
767 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(0));
768 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(1));
769}
770
771test "Vec2D init overflow" {
772 const allocator = testing.allocator;
773 try expectError(
774 error.Overflow,
775 Vec2D(i32).init(allocator, 1, .{ math.maxInt(usize), math.maxInt(usize) }),
776 );
777}
778
779test "Vec2D fill" {
780 const allocator = testing.allocator;
781 var vec2d = try Vec2D(i32).init(allocator, 0, .{ 2, 3 });
782 defer vec2d.deinit(allocator);
783
784 vec2d.fill(7);
785
786 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(0));
787 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(1));
788}
789
790test "Vec2D get" {
791 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
792 const vec2d = Vec2D(i32){
793 .data = &data,
794 .cols = 2,
84 };795 };
796
797 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
798 try expectEqualSlices(i32, &.{ 2, 3 }, try vec2d.get(1));
799 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
800 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
801}
802
803test "Vec2D getMut" {
804 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
805 var vec2d = Vec2D(i32){
806 .data = &data,
807 .cols = 2,
808 };
809
810 const row = try vec2d.getMut(1);
811 row[1] = 9;
812
813 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
814 // (1, 1) should be 9.
815 try expectEqualSlices(i32, &.{ 2, 9 }, try vec2d.get(1));
816 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
817 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
818}
819
820test "Vec2D get multiplication overflow" {
821 const allocator = testing.allocator;
822 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 4 });
823 defer matrix.deinit(allocator);
824
825 const row = (math.maxInt(usize) / 4) + 1;
826 try expectError(error.Overflow, matrix.get(row));
827 try expectError(error.Overflow, matrix.getMut(row));
828}
829
830test "Vec2D get addition overflow" {
831 const allocator = testing.allocator;
832 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 5 });
833 defer matrix.deinit(allocator);
834
835 const row = math.maxInt(usize) / 5;
836 try expectError(error.Overflow, matrix.get(row));
837 try expectError(error.Overflow, matrix.getMut(row));
838}
839
840fn testDecompress(compressed: []const u8) ![]u8 {
841 const allocator = std.testing.allocator;
842 var br: std.io.BufferedReader = undefined;
843 br.initFixed(compressed);
844 var decompressor = try Decompress.initOptions(allocator, &br, .{});
845 defer decompressor.deinit();
846 const reader = decompressor.reader();
847 return reader.readAllAlloc(allocator, std.math.maxInt(usize));
848}
849
850fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
851 const allocator = std.testing.allocator;
852 const decomp = try testDecompress(compressed);
853 defer allocator.free(decomp);
854 try std.testing.expectEqualSlices(u8, expected, decomp);
855}
856
857fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
858 return std.testing.expectError(expected, testDecompress(compressed));
859}
860
861test "decompress empty world" {
862 try testDecompressEqual(
863 "",
864 &[_]u8{
865 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff,
866 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00,
867 },
868 );
869}
870
871test "decompress hello world" {
872 try testDecompressEqual(
873 "Hello world\n",
874 &[_]u8{
875 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
876 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
877 0xa5, 0xb0, 0x00,
878 },
879 );
880}
881
882test "decompress huge dict" {
883 try testDecompressEqual(
884 "Hello world\n",
885 &[_]u8{
886 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
887 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
888 0xa5, 0xb0, 0x00,
889 },
890 );
891}
892
893test "unknown size with end of payload marker" {
894 try testDecompressEqual(
895 "Hello\nWorld!\n",
896 @embedFile("testdata/good-unknown_size-with_eopm.lzma"),
897 );
898}
899
900test "known size without end of payload marker" {
901 try testDecompressEqual(
902 "Hello\nWorld!\n",
903 @embedFile("testdata/good-known_size-without_eopm.lzma"),
904 );
905}
906
907test "known size with end of payload marker" {
908 try testDecompressEqual(
909 "Hello\nWorld!\n",
910 @embedFile("testdata/good-known_size-with_eopm.lzma"),
911 );
912}
913
914test "too big uncompressed size in header" {
915 try testDecompressError(
916 error.CorruptInput,
917 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
918 );
919}
920
921test "too small uncompressed size in header" {
922 try testDecompressError(
923 error.CorruptInput,
924 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
925 );
85}926}
86927
87test {928test "reading one byte" {
88 _ = @import("lzma/test.zig");929 const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");
89 _ = @import("lzma/vec2d.zig");930 var br: std.io.BufferedReader = undefined;
931 br.initFixed(compressed);
932 var decompressor = try Decompress.initOptions(std.testing.allocator, &br, .{});
933 defer decompressor.deinit();
934 var buffer = [1]u8{0};
935 _ = try decompressor.read(buffer[0..]);
90}936}
lib/std/compress/lzma/decode.zig deleted-539
...@@ -1,539 +0,0 @@
1const std = @import("../../std.zig");
2const assert = std.debug.assert;
3const math = std.math;
4const Allocator = std.mem.Allocator;
5
6pub const lzbuffer = @import("decode/lzbuffer.zig");
7
8const LzCircularBuffer = lzbuffer.LzCircularBuffer;
9const Vec2D = @import("vec2d.zig").Vec2D;
10
11pub const RangeDecoder = struct {
12 range: u32,
13 code: u32,
14
15 pub fn init(br: *std.io.BufferedReader) !RangeDecoder {
16 const reserved = try br.takeByte();
17 if (reserved != 0) {
18 return error.CorruptInput;
19 }
20 return .{
21 .range = 0xFFFF_FFFF,
22 .code = try br.readInt(u32, .big),
23 };
24 }
25
26 pub inline fn isFinished(self: RangeDecoder) bool {
27 return self.code == 0;
28 }
29
30 inline fn normalize(self: *RangeDecoder, br: *std.io.BufferedReader) !void {
31 if (self.range < 0x0100_0000) {
32 self.range <<= 8;
33 self.code = (self.code << 8) ^ @as(u32, try br.takeByte());
34 }
35 }
36
37 inline fn getBit(self: *RangeDecoder, br: *std.io.BufferedReader) !bool {
38 self.range >>= 1;
39
40 const bit = self.code >= self.range;
41 if (bit)
42 self.code -= self.range;
43
44 try self.normalize(br);
45 return bit;
46 }
47
48 pub fn get(self: *RangeDecoder, br: *std.io.BufferedReader, count: usize) !u32 {
49 var result: u32 = 0;
50 var i: usize = 0;
51 while (i < count) : (i += 1)
52 result = (result << 1) ^ @intFromBool(try self.getBit(br));
53 return result;
54 }
55
56 pub inline fn decodeBit(self: *RangeDecoder, br: *std.io.BufferedReader, prob: *u16, update: bool) !bool {
57 const bound = (self.range >> 11) * prob.*;
58
59 if (self.code < bound) {
60 if (update)
61 prob.* += (0x800 - prob.*) >> 5;
62 self.range = bound;
63
64 try self.normalize(br);
65 return false;
66 } else {
67 if (update)
68 prob.* -= prob.* >> 5;
69 self.code -= bound;
70 self.range -= bound;
71
72 try self.normalize(br);
73 return true;
74 }
75 }
76
77 fn parseBitTree(
78 self: *RangeDecoder,
79 br: *std.io.BufferedReader,
80 num_bits: u5,
81 probs: []u16,
82 update: bool,
83 ) !u32 {
84 var tmp: u32 = 1;
85 var i: @TypeOf(num_bits) = 0;
86 while (i < num_bits) : (i += 1) {
87 const bit = try self.decodeBit(br, &probs[tmp], update);
88 tmp = (tmp << 1) ^ @intFromBool(bit);
89 }
90 return tmp - (@as(u32, 1) << num_bits);
91 }
92
93 pub fn parseReverseBitTree(
94 self: *RangeDecoder,
95 br: *std.io.BufferedReader,
96 num_bits: u5,
97 probs: []u16,
98 offset: usize,
99 update: bool,
100 ) !u32 {
101 var result: u32 = 0;
102 var tmp: usize = 1;
103 var i: @TypeOf(num_bits) = 0;
104 while (i < num_bits) : (i += 1) {
105 const bit = @intFromBool(try self.decodeBit(br, &probs[offset + tmp], update));
106 tmp = (tmp << 1) ^ bit;
107 result ^= @as(u32, bit) << i;
108 }
109 return result;
110 }
111};
112
113pub fn BitTree(comptime num_bits: usize) type {
114 return struct {
115 probs: [1 << num_bits]u16 = @splat(0x400),
116
117 const Self = @This();
118
119 pub fn parse(
120 self: *Self,
121 br: *std.io.BufferedReader,
122 decoder: *RangeDecoder,
123 update: bool,
124 ) !u32 {
125 return decoder.parseBitTree(br, num_bits, &self.probs, update);
126 }
127
128 pub fn parseReverse(
129 self: *Self,
130 br: *std.io.BufferedReader,
131 decoder: *RangeDecoder,
132 update: bool,
133 ) !u32 {
134 return decoder.parseReverseBitTree(br, num_bits, &self.probs, 0, update);
135 }
136
137 pub fn reset(self: *Self) void {
138 @memset(&self.probs, 0x400);
139 }
140 };
141}
142
143pub const LenDecoder = struct {
144 choice: u16 = 0x400,
145 choice2: u16 = 0x400,
146 low_coder: [16]BitTree(3) = @splat(.{}),
147 mid_coder: [16]BitTree(3) = @splat(.{}),
148 high_coder: BitTree(8) = .{},
149
150 pub fn decode(
151 self: *LenDecoder,
152 br: *std.io.BufferedReader,
153 decoder: *RangeDecoder,
154 pos_state: usize,
155 update: bool,
156 ) !usize {
157 if (!try decoder.decodeBit(br, &self.choice, update)) {
158 return @as(usize, try self.low_coder[pos_state].parse(br, decoder, update));
159 } else if (!try decoder.decodeBit(br, &self.choice2, update)) {
160 return @as(usize, try self.mid_coder[pos_state].parse(br, decoder, update)) + 8;
161 } else {
162 return @as(usize, try self.high_coder.parse(br, decoder, update)) + 16;
163 }
164 }
165
166 pub fn reset(self: *LenDecoder) void {
167 self.choice = 0x400;
168 self.choice2 = 0x400;
169 for (&self.low_coder) |*t| t.reset();
170 for (&self.mid_coder) |*t| t.reset();
171 self.high_coder.reset();
172 }
173};
174
175pub const Options = struct {
176 unpacked_size: UnpackedSize = .read_from_header,
177 memlimit: ?usize = null,
178 allow_incomplete: bool = false,
179};
180
181pub const UnpackedSize = union(enum) {
182 read_from_header,
183 read_header_but_use_provided: ?u64,
184 use_provided: ?u64,
185};
186
187const ProcessingStatus = enum {
188 continue_,
189 finished,
190};
191
192pub const Properties = struct {
193 lc: u4,
194 lp: u3,
195 pb: u3,
196
197 fn validate(self: Properties) void {
198 assert(self.lc <= 8);
199 assert(self.lp <= 4);
200 assert(self.pb <= 4);
201 }
202};
203
204pub const Params = struct {
205 properties: Properties,
206 dict_size: u32,
207 unpacked_size: ?u64,
208
209 pub fn readHeader(reader: anytype, options: Options) !Params {
210 var props = try reader.readByte();
211 if (props >= 225) {
212 return error.CorruptInput;
213 }
214
215 const lc = @as(u4, @intCast(props % 9));
216 props /= 9;
217 const lp = @as(u3, @intCast(props % 5));
218 props /= 5;
219 const pb = @as(u3, @intCast(props));
220
221 const dict_size_provided = try reader.readInt(u32, .little);
222 const dict_size = @max(0x1000, dict_size_provided);
223
224 const unpacked_size = switch (options.unpacked_size) {
225 .read_from_header => blk: {
226 const unpacked_size_provided = try reader.readInt(u64, .little);
227 const marker_mandatory = unpacked_size_provided == 0xFFFF_FFFF_FFFF_FFFF;
228 break :blk if (marker_mandatory)
229 null
230 else
231 unpacked_size_provided;
232 },
233 .read_header_but_use_provided => |x| blk: {
234 _ = try reader.readInt(u64, .little);
235 break :blk x;
236 },
237 .use_provided => |x| x,
238 };
239
240 return Params{
241 .properties = Properties{ .lc = lc, .lp = lp, .pb = pb },
242 .dict_size = dict_size,
243 .unpacked_size = unpacked_size,
244 };
245 }
246};
247
248pub const DecoderState = struct {
249 lzma_props: Properties,
250 unpacked_size: ?u64,
251 literal_probs: Vec2D(u16),
252 pos_slot_decoder: [4]BitTree(6),
253 align_decoder: BitTree(4),
254 pos_decoders: [115]u16,
255 is_match: [192]u16,
256 is_rep: [12]u16,
257 is_rep_g0: [12]u16,
258 is_rep_g1: [12]u16,
259 is_rep_g2: [12]u16,
260 is_rep_0long: [192]u16,
261 state: usize,
262 rep: [4]usize,
263 len_decoder: LenDecoder,
264 rep_len_decoder: LenDecoder,
265
266 pub fn init(
267 allocator: Allocator,
268 lzma_props: Properties,
269 unpacked_size: ?u64,
270 ) !DecoderState {
271 return .{
272 .lzma_props = lzma_props,
273 .unpacked_size = unpacked_size,
274 .literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (lzma_props.lc + lzma_props.lp), 0x300 }),
275 .pos_slot_decoder = @splat(.{}),
276 .align_decoder = .{},
277 .pos_decoders = @splat(0x400),
278 .is_match = @splat(0x400),
279 .is_rep = @splat(0x400),
280 .is_rep_g0 = @splat(0x400),
281 .is_rep_g1 = @splat(0x400),
282 .is_rep_g2 = @splat(0x400),
283 .is_rep_0long = @splat(0x400),
284 .state = 0,
285 .rep = @splat(0),
286 .len_decoder = .{},
287 .rep_len_decoder = .{},
288 };
289 }
290
291 pub fn deinit(self: *DecoderState, allocator: Allocator) void {
292 self.literal_probs.deinit(allocator);
293 self.* = undefined;
294 }
295
296 pub fn resetState(self: *DecoderState, allocator: Allocator, new_props: Properties) !void {
297 new_props.validate();
298 if (self.lzma_props.lc + self.lzma_props.lp == new_props.lc + new_props.lp) {
299 self.literal_probs.fill(0x400);
300 } else {
301 self.literal_probs.deinit(allocator);
302 self.literal_probs = try Vec2D(u16).init(allocator, 0x400, .{ @as(usize, 1) << (new_props.lc + new_props.lp), 0x300 });
303 }
304
305 self.lzma_props = new_props;
306 for (&self.pos_slot_decoder) |*t| t.reset();
307 self.align_decoder.reset();
308 self.pos_decoders = @splat(0x400);
309 self.is_match = @splat(0x400);
310 self.is_rep = @splat(0x400);
311 self.is_rep_g0 = @splat(0x400);
312 self.is_rep_g1 = @splat(0x400);
313 self.is_rep_g2 = @splat(0x400);
314 self.is_rep_0long = @splat(0x400);
315 self.state = 0;
316 self.rep = @splat(0);
317 self.len_decoder.reset();
318 self.rep_len_decoder.reset();
319 }
320
321 fn processNextInner(
322 self: *DecoderState,
323 allocator: Allocator,
324 reader: anytype,
325 writer: anytype,
326 buffer: anytype,
327 decoder: *RangeDecoder,
328 update: bool,
329 ) !ProcessingStatus {
330 const pos_state = buffer.len & ((@as(usize, 1) << self.lzma_props.pb) - 1);
331
332 if (!try decoder.decodeBit(
333 reader,
334 &self.is_match[(self.state << 4) + pos_state],
335 update,
336 )) {
337 const byte: u8 = try self.decodeLiteral(reader, buffer, decoder, update);
338
339 if (update) {
340 try buffer.appendLiteral(allocator, byte, writer);
341
342 self.state = if (self.state < 4)
343 0
344 else if (self.state < 10)
345 self.state - 3
346 else
347 self.state - 6;
348 }
349 return .continue_;
350 }
351
352 var len: usize = undefined;
353 if (try decoder.decodeBit(reader, &self.is_rep[self.state], update)) {
354 if (!try decoder.decodeBit(reader, &self.is_rep_g0[self.state], update)) {
355 if (!try decoder.decodeBit(
356 reader,
357 &self.is_rep_0long[(self.state << 4) + pos_state],
358 update,
359 )) {
360 if (update) {
361 self.state = if (self.state < 7) 9 else 11;
362 const dist = self.rep[0] + 1;
363 try buffer.appendLz(allocator, 1, dist, writer);
364 }
365 return .continue_;
366 }
367 } else {
368 const idx: usize = if (!try decoder.decodeBit(reader, &self.is_rep_g1[self.state], update))
369 1
370 else if (!try decoder.decodeBit(reader, &self.is_rep_g2[self.state], update))
371 2
372 else
373 3;
374 if (update) {
375 const dist = self.rep[idx];
376 var i = idx;
377 while (i > 0) : (i -= 1) {
378 self.rep[i] = self.rep[i - 1];
379 }
380 self.rep[0] = dist;
381 }
382 }
383
384 len = try self.rep_len_decoder.decode(reader, decoder, pos_state, update);
385
386 if (update) {
387 self.state = if (self.state < 7) 8 else 11;
388 }
389 } else {
390 if (update) {
391 self.rep[3] = self.rep[2];
392 self.rep[2] = self.rep[1];
393 self.rep[1] = self.rep[0];
394 }
395
396 len = try self.len_decoder.decode(reader, decoder, pos_state, update);
397
398 if (update) {
399 self.state = if (self.state < 7) 7 else 10;
400 }
401
402 const rep_0 = try self.decodeDistance(reader, decoder, len, update);
403
404 if (update) {
405 self.rep[0] = rep_0;
406 if (self.rep[0] == 0xFFFF_FFFF) {
407 if (decoder.isFinished()) {
408 return .finished;
409 }
410 return error.CorruptInput;
411 }
412 }
413 }
414
415 if (update) {
416 len += 2;
417
418 const dist = self.rep[0] + 1;
419 try buffer.appendLz(allocator, len, dist, writer);
420 }
421
422 return .continue_;
423 }
424
425 fn processNext(
426 self: *DecoderState,
427 allocator: Allocator,
428 reader: anytype,
429 writer: anytype,
430 buffer: anytype,
431 decoder: *RangeDecoder,
432 ) !ProcessingStatus {
433 return self.processNextInner(allocator, reader, writer, buffer, decoder, true);
434 }
435
436 pub fn process(
437 self: *DecoderState,
438 allocator: Allocator,
439 reader: anytype,
440 writer: anytype,
441 buffer: anytype,
442 decoder: *RangeDecoder,
443 ) !ProcessingStatus {
444 process_next: {
445 if (self.unpacked_size) |unpacked_size| {
446 if (buffer.len >= unpacked_size) {
447 break :process_next;
448 }
449 } else if (decoder.isFinished()) {
450 break :process_next;
451 }
452
453 switch (try self.processNext(allocator, reader, writer, buffer, decoder)) {
454 .continue_ => return .continue_,
455 .finished => break :process_next,
456 }
457 }
458
459 if (self.unpacked_size) |unpacked_size| {
460 if (buffer.len != unpacked_size) {
461 return error.CorruptInput;
462 }
463 }
464
465 return .finished;
466 }
467
468 fn decodeLiteral(
469 self: *DecoderState,
470 reader: anytype,
471 buffer: anytype,
472 decoder: *RangeDecoder,
473 update: bool,
474 ) !u8 {
475 const def_prev_byte = 0;
476 const prev_byte = @as(usize, buffer.lastOr(def_prev_byte));
477
478 var result: usize = 1;
479 const lit_state = ((buffer.len & ((@as(usize, 1) << self.lzma_props.lp) - 1)) << self.lzma_props.lc) +
480 (prev_byte >> (8 - self.lzma_props.lc));
481 const probs = try self.literal_probs.getMut(lit_state);
482
483 if (self.state >= 7) {
484 var match_byte = @as(usize, try buffer.lastN(self.rep[0] + 1));
485
486 while (result < 0x100) {
487 const match_bit = (match_byte >> 7) & 1;
488 match_byte <<= 1;
489 const bit = @intFromBool(try decoder.decodeBit(
490 reader,
491 &probs[((@as(usize, 1) + match_bit) << 8) + result],
492 update,
493 ));
494 result = (result << 1) ^ bit;
495 if (match_bit != bit) {
496 break;
497 }
498 }
499 }
500
501 while (result < 0x100) {
502 result = (result << 1) ^ @intFromBool(try decoder.decodeBit(reader, &probs[result], update));
503 }
504
505 return @as(u8, @truncate(result - 0x100));
506 }
507
508 fn decodeDistance(
509 self: *DecoderState,
510 reader: anytype,
511 decoder: *RangeDecoder,
512 length: usize,
513 update: bool,
514 ) !usize {
515 const len_state = if (length > 3) 3 else length;
516
517 const pos_slot = @as(usize, try self.pos_slot_decoder[len_state].parse(reader, decoder, update));
518 if (pos_slot < 4)
519 return pos_slot;
520
521 const num_direct_bits = @as(u5, @intCast((pos_slot >> 1) - 1));
522 var result = (2 ^ (pos_slot & 1)) << num_direct_bits;
523
524 if (pos_slot < 14) {
525 result += try decoder.parseReverseBitTree(
526 reader,
527 num_direct_bits,
528 &self.pos_decoders,
529 result - pos_slot,
530 update,
531 );
532 } else {
533 result += @as(usize, try decoder.get(reader, num_direct_bits - 4)) << 4;
534 result += try self.align_decoder.parseReverse(reader, decoder, update);
535 }
536
537 return result;
538 }
539};
lib/std/compress/lzma/decode/lzbuffer.zig deleted-228
...@@ -1,228 +0,0 @@
1const std = @import("../../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5const ArrayListUnmanaged = std.ArrayListUnmanaged;
6
7/// An accumulating buffer for LZ sequences
8pub const LzAccumBuffer = struct {
9 /// Buffer
10 buf: ArrayListUnmanaged(u8),
11
12 /// Buffer memory limit
13 memlimit: usize,
14
15 /// Total number of bytes sent through the buffer
16 len: usize,
17
18 const Self = @This();
19
20 pub fn init(memlimit: usize) Self {
21 return Self{
22 .buf = .{},
23 .memlimit = memlimit,
24 .len = 0,
25 };
26 }
27
28 pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
29 try self.buf.append(allocator, byte);
30 self.len += 1;
31 }
32
33 /// Reset the internal dictionary
34 pub fn reset(self: *Self, writer: anytype) !void {
35 try writer.writeAll(self.buf.items);
36 self.buf.clearRetainingCapacity();
37 self.len = 0;
38 }
39
40 /// Retrieve the last byte or return a default
41 pub fn lastOr(self: Self, lit: u8) u8 {
42 const buf_len = self.buf.items.len;
43 return if (buf_len == 0)
44 lit
45 else
46 self.buf.items[buf_len - 1];
47 }
48
49 /// Retrieve the n-th last byte
50 pub fn lastN(self: Self, dist: usize) !u8 {
51 const buf_len = self.buf.items.len;
52 if (dist > buf_len) {
53 return error.CorruptInput;
54 }
55
56 return self.buf.items[buf_len - dist];
57 }
58
59 /// Append a literal
60 pub fn appendLiteral(
61 self: *Self,
62 allocator: Allocator,
63 lit: u8,
64 writer: anytype,
65 ) !void {
66 _ = writer;
67 if (self.len >= self.memlimit) {
68 return error.CorruptInput;
69 }
70 try self.buf.append(allocator, lit);
71 self.len += 1;
72 }
73
74 /// Fetch an LZ sequence (length, distance) from inside the buffer
75 pub fn appendLz(
76 self: *Self,
77 allocator: Allocator,
78 len: usize,
79 dist: usize,
80 writer: anytype,
81 ) !void {
82 _ = writer;
83
84 const buf_len = self.buf.items.len;
85 if (dist > buf_len) {
86 return error.CorruptInput;
87 }
88
89 var offset = buf_len - dist;
90 var i: usize = 0;
91 while (i < len) : (i += 1) {
92 const x = self.buf.items[offset];
93 try self.buf.append(allocator, x);
94 offset += 1;
95 }
96 self.len += len;
97 }
98
99 pub fn finish(self: *Self, writer: anytype) !void {
100 try writer.writeAll(self.buf.items);
101 self.buf.clearRetainingCapacity();
102 }
103
104 pub fn deinit(self: *Self, allocator: Allocator) void {
105 self.buf.deinit(allocator);
106 self.* = undefined;
107 }
108};
109
110/// A circular buffer for LZ sequences
111pub const LzCircularBuffer = struct {
112 /// Circular buffer
113 buf: ArrayListUnmanaged(u8),
114
115 /// Length of the buffer
116 dict_size: usize,
117
118 /// Buffer memory limit
119 memlimit: usize,
120
121 /// Current position
122 cursor: usize,
123
124 /// Total number of bytes sent through the buffer
125 len: usize,
126
127 const Self = @This();
128
129 pub fn init(dict_size: usize, memlimit: usize) Self {
130 return Self{
131 .buf = .{},
132 .dict_size = dict_size,
133 .memlimit = memlimit,
134 .cursor = 0,
135 .len = 0,
136 };
137 }
138
139 pub fn get(self: Self, index: usize) u8 {
140 return if (0 <= index and index < self.buf.items.len)
141 self.buf.items[index]
142 else
143 0;
144 }
145
146 pub fn set(self: *Self, allocator: Allocator, index: usize, value: u8) !void {
147 if (index >= self.memlimit) {
148 return error.CorruptInput;
149 }
150 try self.buf.ensureTotalCapacity(allocator, index + 1);
151 while (self.buf.items.len < index) {
152 self.buf.appendAssumeCapacity(0);
153 }
154 self.buf.appendAssumeCapacity(value);
155 }
156
157 /// Retrieve the last byte or return a default
158 pub fn lastOr(self: Self, lit: u8) u8 {
159 return if (self.len == 0)
160 lit
161 else
162 self.get((self.dict_size + self.cursor - 1) % self.dict_size);
163 }
164
165 /// Retrieve the n-th last byte
166 pub fn lastN(self: Self, dist: usize) !u8 {
167 if (dist > self.dict_size or dist > self.len) {
168 return error.CorruptInput;
169 }
170
171 const offset = (self.dict_size + self.cursor - dist) % self.dict_size;
172 return self.get(offset);
173 }
174
175 /// Append a literal
176 pub fn appendLiteral(
177 self: *Self,
178 allocator: Allocator,
179 lit: u8,
180 writer: anytype,
181 ) !void {
182 try self.set(allocator, self.cursor, lit);
183 self.cursor += 1;
184 self.len += 1;
185
186 // Flush the circular buffer to the output
187 if (self.cursor == self.dict_size) {
188 try writer.writeAll(self.buf.items);
189 self.cursor = 0;
190 }
191 }
192
193 /// Fetch an LZ sequence (length, distance) from inside the buffer
194 pub fn appendLz(
195 self: *Self,
196 allocator: Allocator,
197 len: usize,
198 dist: usize,
199 writer: anytype,
200 ) !void {
201 if (dist > self.dict_size or dist > self.len) {
202 return error.CorruptInput;
203 }
204
205 var offset = (self.dict_size + self.cursor - dist) % self.dict_size;
206 var i: usize = 0;
207 while (i < len) : (i += 1) {
208 const x = self.get(offset);
209 try self.appendLiteral(allocator, x, writer);
210 offset += 1;
211 if (offset == self.dict_size) {
212 offset = 0;
213 }
214 }
215 }
216
217 pub fn finish(self: *Self, writer: anytype) !void {
218 if (self.cursor > 0) {
219 try writer.writeAll(self.buf.items[0..self.cursor]);
220 self.cursor = 0;
221 }
222 }
223
224 pub fn deinit(self: *Self, allocator: Allocator) void {
225 self.buf.deinit(allocator);
226 self.* = undefined;
227 }
228};
lib/std/compress/lzma/test.zig deleted-99
...@@ -1,99 +0,0 @@
1const std = @import("../../std.zig");
2const lzma = @import("../lzma.zig");
3
4fn testDecompress(compressed: []const u8) ![]u8 {
5 const allocator = std.testing.allocator;
6 var stream = std.io.fixedBufferStream(compressed);
7 var decompressor = try lzma.decompress(allocator, stream.reader());
8 defer decompressor.deinit();
9 const reader = decompressor.reader();
10 return reader.readAllAlloc(allocator, std.math.maxInt(usize));
11}
12
13fn testDecompressEqual(expected: []const u8, compressed: []const u8) !void {
14 const allocator = std.testing.allocator;
15 const decomp = try testDecompress(compressed);
16 defer allocator.free(decomp);
17 try std.testing.expectEqualSlices(u8, expected, decomp);
18}
19
20fn testDecompressError(expected: anyerror, compressed: []const u8) !void {
21 return std.testing.expectError(expected, testDecompress(compressed));
22}
23
24test "decompress empty world" {
25 try testDecompressEqual(
26 "",
27 &[_]u8{
28 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x83, 0xff,
29 0xfb, 0xff, 0xff, 0xc0, 0x00, 0x00, 0x00,
30 },
31 );
32}
33
34test "decompress hello world" {
35 try testDecompressEqual(
36 "Hello world\n",
37 &[_]u8{
38 0x5d, 0x00, 0x00, 0x80, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
39 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
40 0xa5, 0xb0, 0x00,
41 },
42 );
43}
44
45test "decompress huge dict" {
46 try testDecompressEqual(
47 "Hello world\n",
48 &[_]u8{
49 0x5d, 0x7f, 0x7f, 0x7f, 0x7f, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0x24, 0x19,
50 0x49, 0x98, 0x6f, 0x10, 0x19, 0xc6, 0xd7, 0x31, 0xeb, 0x36, 0x50, 0xb2, 0x98, 0x48, 0xff, 0xfe,
51 0xa5, 0xb0, 0x00,
52 },
53 );
54}
55
56test "unknown size with end of payload marker" {
57 try testDecompressEqual(
58 "Hello\nWorld!\n",
59 @embedFile("testdata/good-unknown_size-with_eopm.lzma"),
60 );
61}
62
63test "known size without end of payload marker" {
64 try testDecompressEqual(
65 "Hello\nWorld!\n",
66 @embedFile("testdata/good-known_size-without_eopm.lzma"),
67 );
68}
69
70test "known size with end of payload marker" {
71 try testDecompressEqual(
72 "Hello\nWorld!\n",
73 @embedFile("testdata/good-known_size-with_eopm.lzma"),
74 );
75}
76
77test "too big uncompressed size in header" {
78 try testDecompressError(
79 error.CorruptInput,
80 @embedFile("testdata/bad-too_big_size-with_eopm.lzma"),
81 );
82}
83
84test "too small uncompressed size in header" {
85 try testDecompressError(
86 error.CorruptInput,
87 @embedFile("testdata/bad-too_small_size-without_eopm-3.lzma"),
88 );
89}
90
91test "reading one byte" {
92 const compressed = @embedFile("testdata/good-known_size-with_eopm.lzma");
93 var stream = std.io.fixedBufferStream(compressed);
94 var decompressor = try lzma.decompress(std.testing.allocator, stream.reader());
95 defer decompressor.deinit();
96
97 var buffer = [1]u8{0};
98 _ = try decompressor.read(buffer[0..]);
99}
lib/std/compress/lzma/vec2d.zig deleted-128
...@@ -1,128 +0,0 @@
1const std = @import("../../std.zig");
2const math = std.math;
3const mem = std.mem;
4const Allocator = std.mem.Allocator;
5
6pub fn Vec2D(comptime T: type) type {
7 return struct {
8 data: []T,
9 cols: usize,
10
11 const Self = @This();
12
13 pub fn init(allocator: Allocator, value: T, size: struct { usize, usize }) !Self {
14 const len = try math.mul(usize, size[0], size[1]);
15 const data = try allocator.alloc(T, len);
16 @memset(data, value);
17 return Self{
18 .data = data,
19 .cols = size[1],
20 };
21 }
22
23 pub fn deinit(self: *Self, allocator: Allocator) void {
24 allocator.free(self.data);
25 self.* = undefined;
26 }
27
28 pub fn fill(self: *Self, value: T) void {
29 @memset(self.data, value);
30 }
31
32 inline fn _get(self: Self, row: usize) ![]T {
33 const start_row = try math.mul(usize, row, self.cols);
34 const end_row = try math.add(usize, start_row, self.cols);
35 return self.data[start_row..end_row];
36 }
37
38 pub fn get(self: Self, row: usize) ![]const T {
39 return self._get(row);
40 }
41
42 pub fn getMut(self: *Self, row: usize) ![]T {
43 return self._get(row);
44 }
45 };
46}
47
48const testing = std.testing;
49const expectEqualSlices = std.testing.expectEqualSlices;
50const expectError = std.testing.expectError;
51
52test "init" {
53 const allocator = testing.allocator;
54 var vec2d = try Vec2D(i32).init(allocator, 1, .{ 2, 3 });
55 defer vec2d.deinit(allocator);
56
57 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(0));
58 try expectEqualSlices(i32, &.{ 1, 1, 1 }, try vec2d.get(1));
59}
60
61test "init overflow" {
62 const allocator = testing.allocator;
63 try expectError(
64 error.Overflow,
65 Vec2D(i32).init(allocator, 1, .{ math.maxInt(usize), math.maxInt(usize) }),
66 );
67}
68
69test "fill" {
70 const allocator = testing.allocator;
71 var vec2d = try Vec2D(i32).init(allocator, 0, .{ 2, 3 });
72 defer vec2d.deinit(allocator);
73
74 vec2d.fill(7);
75
76 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(0));
77 try expectEqualSlices(i32, &.{ 7, 7, 7 }, try vec2d.get(1));
78}
79
80test "get" {
81 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
82 const vec2d = Vec2D(i32){
83 .data = &data,
84 .cols = 2,
85 };
86
87 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
88 try expectEqualSlices(i32, &.{ 2, 3 }, try vec2d.get(1));
89 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
90 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
91}
92
93test "getMut" {
94 var data = [_]i32{ 0, 1, 2, 3, 4, 5, 6, 7 };
95 var vec2d = Vec2D(i32){
96 .data = &data,
97 .cols = 2,
98 };
99
100 const row = try vec2d.getMut(1);
101 row[1] = 9;
102
103 try expectEqualSlices(i32, &.{ 0, 1 }, try vec2d.get(0));
104 // (1, 1) should be 9.
105 try expectEqualSlices(i32, &.{ 2, 9 }, try vec2d.get(1));
106 try expectEqualSlices(i32, &.{ 4, 5 }, try vec2d.get(2));
107 try expectEqualSlices(i32, &.{ 6, 7 }, try vec2d.get(3));
108}
109
110test "get multiplication overflow" {
111 const allocator = testing.allocator;
112 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 4 });
113 defer matrix.deinit(allocator);
114
115 const row = (math.maxInt(usize) / 4) + 1;
116 try expectError(error.Overflow, matrix.get(row));
117 try expectError(error.Overflow, matrix.getMut(row));
118}
119
120test "get addition overflow" {
121 const allocator = testing.allocator;
122 var matrix = try Vec2D(i32).init(allocator, 0, .{ 3, 5 });
123 defer matrix.deinit(allocator);
124
125 const row = math.maxInt(usize) / 5;
126 try expectError(error.Overflow, matrix.get(row));
127 try expectError(error.Overflow, matrix.getMut(row));
128}
lib/std/compress/lzma2.zig+268-7
...@@ -1,15 +1,276 @@...@@ -1,15 +1,276 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;2const Allocator = std.mem.Allocator;
3const lzma = std.compress.lzma;
34
4pub const decode = @import("lzma2/decode.zig");5pub fn decompress(gpa: Allocator, reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) anyerror!void {
56 var decoder = try Decode.init(gpa);
6pub fn decompress(allocator: Allocator, reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {7 defer decoder.deinit(gpa);
7 var decoder = try decode.Decoder.init(allocator);8 return decoder.decompress(gpa, reader, writer);
8 defer decoder.deinit(allocator);
9 return decoder.decompress(allocator, reader, writer);
10}9}
1110
12test {11pub const Decode = struct {
12 lzma1: lzma.Decode,
13
14 pub fn init(allocator: Allocator) !Decode {
15 return .{
16 .lzma1 = try lzma.Decode.init(
17 allocator,
18 .{
19 .lc = 0,
20 .lp = 0,
21 .pb = 0,
22 },
23 null,
24 ),
25 };
26 }
27
28 pub fn deinit(self: *Decode, allocator: Allocator) void {
29 self.lzma1.deinit(allocator);
30 self.* = undefined;
31 }
32
33 pub fn decompress(
34 self: *Decode,
35 allocator: Allocator,
36 reader: *std.io.BufferedReader,
37 writer: *std.io.BufferedWriter,
38 ) !void {
39 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
40 defer accum.deinit(allocator);
41
42 while (true) {
43 const status = try reader.takeByte();
44
45 switch (status) {
46 0 => break,
47 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
48 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
49 else => try self.parseLzma(allocator, reader, writer, &accum, status),
50 }
51 }
52
53 try accum.finish(writer);
54 }
55
56 fn parseLzma(
57 self: *Decode,
58 allocator: Allocator,
59 br: *std.io.BufferedReader,
60 writer: *std.io.BufferedWriter,
61 accum: *LzAccumBuffer,
62 status: u8,
63 ) !void {
64 if (status & 0x80 == 0) {
65 return error.CorruptInput;
66 }
67
68 const Reset = struct {
69 dict: bool,
70 state: bool,
71 props: bool,
72 };
73
74 const reset = switch ((status >> 5) & 0x3) {
75 0 => Reset{
76 .dict = false,
77 .state = false,
78 .props = false,
79 },
80 1 => Reset{
81 .dict = false,
82 .state = true,
83 .props = false,
84 },
85 2 => Reset{
86 .dict = false,
87 .state = true,
88 .props = true,
89 },
90 3 => Reset{
91 .dict = true,
92 .state = true,
93 .props = true,
94 },
95 else => unreachable,
96 };
97
98 const unpacked_size = blk: {
99 var tmp: u64 = status & 0x1F;
100 tmp <<= 16;
101 tmp |= try br.takeInt(u16, .big);
102 break :blk tmp + 1;
103 };
104
105 const packed_size = blk: {
106 const tmp: u17 = try br.takeInt(u16, .big);
107 break :blk tmp + 1;
108 };
109
110 if (reset.dict) {
111 try accum.reset(writer);
112 }
113
114 if (reset.state) {
115 var new_props = self.lzma1.properties;
116
117 if (reset.props) {
118 var props = try br.takeByte();
119 if (props >= 225) {
120 return error.CorruptInput;
121 }
122
123 const lc = @as(u4, @intCast(props % 9));
124 props /= 9;
125 const lp = @as(u3, @intCast(props % 5));
126 props /= 5;
127 const pb = @as(u3, @intCast(props));
128
129 if (lc + lp > 4) {
130 return error.CorruptInput;
131 }
132
133 new_props = .{ .lc = lc, .lp = lp, .pb = pb };
134 }
135
136 try self.lzma1.resetState(allocator, new_props);
137 }
138
139 self.lzma1.unpacked_size = unpacked_size + accum.len;
140
141 var range_decoder: lzma.RangeDecoder = undefined;
142 var bytes_read = try lzma.RangeDecoder.init(br);
143 while (try self.lzma1.process(allocator, br, writer, accum, &range_decoder, &bytes_read) == .cont) {}
144
145 if (bytes_read != packed_size) {
146 return error.CorruptInput;
147 }
148 }
149
150 fn parseUncompressed(
151 allocator: Allocator,
152 reader: *std.io.BufferedReader,
153 writer: *std.io.BufferedWriter,
154 accum: *LzAccumBuffer,
155 reset_dict: bool,
156 ) !void {
157 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
158
159 if (reset_dict) {
160 try accum.reset(writer);
161 }
162
163 var i: @TypeOf(unpacked_size) = 0;
164 while (i < unpacked_size) : (i += 1) {
165 try accum.appendByte(allocator, try reader.takeByte());
166 }
167 }
168};
169
170/// An accumulating buffer for LZ sequences
171const LzAccumBuffer = struct {
172 /// Buffer
173 buf: std.ArrayListUnmanaged(u8),
174
175 /// Buffer memory limit
176 memlimit: usize,
177
178 /// Total number of bytes sent through the buffer
179 len: usize,
180
181 const Self = @This();
182
183 pub fn init(memlimit: usize) Self {
184 return Self{
185 .buf = .{},
186 .memlimit = memlimit,
187 .len = 0,
188 };
189 }
190
191 pub fn appendByte(self: *Self, allocator: Allocator, byte: u8) !void {
192 try self.buf.append(allocator, byte);
193 self.len += 1;
194 }
195
196 /// Reset the internal dictionary
197 pub fn reset(self: *Self, writer: anytype) !void {
198 try writer.writeAll(self.buf.items);
199 self.buf.clearRetainingCapacity();
200 self.len = 0;
201 }
202
203 /// Retrieve the last byte or return a default
204 pub fn lastOr(self: Self, lit: u8) u8 {
205 const buf_len = self.buf.items.len;
206 return if (buf_len == 0)
207 lit
208 else
209 self.buf.items[buf_len - 1];
210 }
211
212 /// Retrieve the n-th last byte
213 pub fn lastN(self: Self, dist: usize) !u8 {
214 const buf_len = self.buf.items.len;
215 if (dist > buf_len) {
216 return error.CorruptInput;
217 }
218
219 return self.buf.items[buf_len - dist];
220 }
221
222 /// Append a literal
223 pub fn appendLiteral(
224 self: *Self,
225 allocator: Allocator,
226 lit: u8,
227 writer: anytype,
228 ) !void {
229 _ = writer;
230 if (self.len >= self.memlimit) {
231 return error.CorruptInput;
232 }
233 try self.buf.append(allocator, lit);
234 self.len += 1;
235 }
236
237 /// Fetch an LZ sequence (length, distance) from inside the buffer
238 pub fn appendLz(
239 self: *Self,
240 allocator: Allocator,
241 len: usize,
242 dist: usize,
243 writer: anytype,
244 ) !void {
245 _ = writer;
246
247 const buf_len = self.buf.items.len;
248 if (dist > buf_len) {
249 return error.CorruptInput;
250 }
251
252 var offset = buf_len - dist;
253 var i: usize = 0;
254 while (i < len) : (i += 1) {
255 const x = self.buf.items[offset];
256 try self.buf.append(allocator, x);
257 offset += 1;
258 }
259 self.len += len;
260 }
261
262 pub fn finish(self: *Self, writer: anytype) !void {
263 try writer.writeAll(self.buf.items);
264 self.buf.clearRetainingCapacity();
265 }
266
267 pub fn deinit(self: *Self, allocator: Allocator) void {
268 self.buf.deinit(allocator);
269 self.* = undefined;
270 }
271};
272
273test decompress {
13 const expected = "Hello\nWorld!\n";274 const expected = "Hello\nWorld!\n";
14 const compressed = [_]u8{275 const compressed = [_]u8{
15 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02,276 0x01, 0x00, 0x05, 0x48, 0x65, 0x6C, 0x6C, 0x6F, 0x0A, 0x02,
lib/std/compress/lzma2/decode.zig deleted-169
...@@ -1,169 +0,0 @@
1const std = @import("../../std.zig");
2const Allocator = std.mem.Allocator;
3
4const lzma = @import("../lzma.zig");
5const DecoderState = lzma.decode.DecoderState;
6const LzAccumBuffer = lzma.decode.lzbuffer.LzAccumBuffer;
7const Properties = lzma.decode.Properties;
8const RangeDecoder = lzma.decode.RangeDecoder;
9
10pub const Decoder = struct {
11 lzma_state: DecoderState,
12
13 pub fn init(allocator: Allocator) !Decoder {
14 return Decoder{
15 .lzma_state = try DecoderState.init(
16 allocator,
17 Properties{
18 .lc = 0,
19 .lp = 0,
20 .pb = 0,
21 },
22 null,
23 ),
24 };
25 }
26
27 pub fn deinit(self: *Decoder, allocator: Allocator) void {
28 self.lzma_state.deinit(allocator);
29 self.* = undefined;
30 }
31
32 pub fn decompress(
33 self: *Decoder,
34 allocator: Allocator,
35 reader: *std.io.BufferedReader,
36 writer: *std.io.BufferedWriter,
37 ) !void {
38 var accum = LzAccumBuffer.init(std.math.maxInt(usize));
39 defer accum.deinit(allocator);
40
41 while (true) {
42 const status = try reader.takeByte();
43
44 switch (status) {
45 0 => break,
46 1 => try parseUncompressed(allocator, reader, writer, &accum, true),
47 2 => try parseUncompressed(allocator, reader, writer, &accum, false),
48 else => try self.parseLzma(allocator, reader, writer, &accum, status),
49 }
50 }
51
52 try accum.finish(writer);
53 }
54
55 fn parseLzma(
56 self: *Decoder,
57 allocator: Allocator,
58 br: *std.io.BufferedReader,
59 writer: *std.io.BufferedWriter,
60 accum: *LzAccumBuffer,
61 status: u8,
62 ) !void {
63 if (status & 0x80 == 0) {
64 return error.CorruptInput;
65 }
66
67 const Reset = struct {
68 dict: bool,
69 state: bool,
70 props: bool,
71 };
72
73 const reset = switch ((status >> 5) & 0x3) {
74 0 => Reset{
75 .dict = false,
76 .state = false,
77 .props = false,
78 },
79 1 => Reset{
80 .dict = false,
81 .state = true,
82 .props = false,
83 },
84 2 => Reset{
85 .dict = false,
86 .state = true,
87 .props = true,
88 },
89 3 => Reset{
90 .dict = true,
91 .state = true,
92 .props = true,
93 },
94 else => unreachable,
95 };
96
97 const unpacked_size = blk: {
98 var tmp: u64 = status & 0x1F;
99 tmp <<= 16;
100 tmp |= try br.takeInt(u16, .big);
101 break :blk tmp + 1;
102 };
103
104 const packed_size = blk: {
105 const tmp: u17 = try br.takeInt(u16, .big);
106 break :blk tmp + 1;
107 };
108
109 if (reset.dict) {
110 try accum.reset(writer);
111 }
112
113 if (reset.state) {
114 var new_props = self.lzma_state.lzma_props;
115
116 if (reset.props) {
117 var props = try br.takeByte();
118 if (props >= 225) {
119 return error.CorruptInput;
120 }
121
122 const lc = @as(u4, @intCast(props % 9));
123 props /= 9;
124 const lp = @as(u3, @intCast(props % 5));
125 props /= 5;
126 const pb = @as(u3, @intCast(props));
127
128 if (lc + lp > 4) {
129 return error.CorruptInput;
130 }
131
132 new_props = Properties{ .lc = lc, .lp = lp, .pb = pb };
133 }
134
135 try self.lzma_state.resetState(allocator, new_props);
136 }
137
138 self.lzma_state.unpacked_size = unpacked_size + accum.len;
139
140 var counter: std.io.CountingReader = .{ .child_reader = br.reader() };
141 var counter_reader = counter.reader().unbuffered();
142
143 var rangecoder = try RangeDecoder.init(&counter_reader);
144 while (try self.lzma_state.process(allocator, counter_reader, writer, accum, &rangecoder) == .continue_) {}
145
146 if (counter.bytes_read != packed_size) {
147 return error.CorruptInput;
148 }
149 }
150
151 fn parseUncompressed(
152 allocator: Allocator,
153 reader: *std.io.BufferedReader,
154 writer: *std.io.BufferedWriter,
155 accum: *LzAccumBuffer,
156 reset_dict: bool,
157 ) !void {
158 const unpacked_size = @as(u17, try reader.takeInt(u16, .big)) + 1;
159
160 if (reset_dict) {
161 try accum.reset(writer);
162 }
163
164 var i: @TypeOf(unpacked_size) = 0;
165 while (i < unpacked_size) : (i += 1) {
166 try accum.appendByte(allocator, try reader.takeByte());
167 }
168 }
169};
lib/std/compress/zstandard.zig+159-163
...@@ -16,191 +16,187 @@ pub const DecompressorOptions = struct {...@@ -16,191 +16,187 @@ pub const DecompressorOptions = struct {
16 pub const default_window_buffer_len = 8 * 1024 * 1024;16 pub const default_window_buffer_len = 8 * 1024 * 1024;
17};17};
1818
19pub fn Decompressor(comptime ReaderType: type) type {19pub const Decompressor = struct {
20 return struct {20 const Self = @This();
21 const Self = @This();21
2222 const table_size_max = types.compressed_block.table_size_max;
23 const table_size_max = types.compressed_block.table_size_max;23
2424 source: std.io.CountingReader,
25 source: std.io.CountingReader(ReaderType),25 state: enum { NewFrame, InFrame, LastBlock },
26 state: enum { NewFrame, InFrame, LastBlock },26 decode_state: decompress.block.DecodeState,
27 decode_state: decompress.block.DecodeState,27 frame_context: decompress.FrameContext,
28 frame_context: decompress.FrameContext,28 buffer: WindowBuffer,
29 buffer: WindowBuffer,29 literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,
30 literal_fse_buffer: [table_size_max.literal]types.compressed_block.Table.Fse,30 match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,
31 match_fse_buffer: [table_size_max.match]types.compressed_block.Table.Fse,31 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,
32 offset_fse_buffer: [table_size_max.offset]types.compressed_block.Table.Fse,32 literals_buffer: [types.block_size_max]u8,
33 literals_buffer: [types.block_size_max]u8,33 sequence_buffer: [types.block_size_max]u8,
34 sequence_buffer: [types.block_size_max]u8,34 verify_checksum: bool,
35 verify_checksum: bool,35 checksum: ?u32,
36 checksum: ?u32,36 current_frame_decompressed_size: usize,
37 current_frame_decompressed_size: usize,37
3838 const WindowBuffer = struct {
39 const WindowBuffer = struct {39 data: []u8 = undefined,
40 data: []u8 = undefined,40 read_index: usize = 0,
41 read_index: usize = 0,41 write_index: usize = 0,
42 write_index: usize = 0,42 };
43 };43
44 pub const Error = anyerror || error{
45 ChecksumFailure,
46 DictionaryIdFlagUnsupported,
47 MalformedBlock,
48 MalformedFrame,
49 OutOfMemory,
50 };
4451
45 pub const Error = ReaderType.Error || error{52 pub fn init(source: *std.io.BufferedReader, options: DecompressorOptions) Self {
46 ChecksumFailure,53 return .{
47 DictionaryIdFlagUnsupported,54 .source = std.io.countingReader(source),
48 MalformedBlock,55 .state = .NewFrame,
49 MalformedFrame,56 .decode_state = undefined,
50 OutOfMemory,57 .frame_context = undefined,
58 .buffer = .{ .data = options.window_buffer },
59 .literal_fse_buffer = undefined,
60 .match_fse_buffer = undefined,
61 .offset_fse_buffer = undefined,
62 .literals_buffer = undefined,
63 .sequence_buffer = undefined,
64 .verify_checksum = options.verify_checksum,
65 .checksum = undefined,
66 .current_frame_decompressed_size = undefined,
51 };67 };
68 }
5269
53 pub const Reader = std.io.Reader(*Self, Error, read);70 fn frameInit(self: *Self) !void {
5471 const source_reader = self.source;
55 pub fn init(source: ReaderType, options: DecompressorOptions) Self {72 switch (try decompress.decodeFrameHeader(source_reader)) {
56 return .{73 .skippable => |header| {
57 .source = std.io.countingReader(source),74 try source_reader.skipBytes(header.frame_size, .{});
58 .state = .NewFrame,75 self.state = .NewFrame;
59 .decode_state = undefined,76 },
60 .frame_context = undefined,77 .zstandard => |header| {
61 .buffer = .{ .data = options.window_buffer },78 const frame_context = try decompress.FrameContext.init(
62 .literal_fse_buffer = undefined,79 header,
63 .match_fse_buffer = undefined,80 self.buffer.data.len,
64 .offset_fse_buffer = undefined,81 self.verify_checksum,
65 .literals_buffer = undefined,82 );
66 .sequence_buffer = undefined,83
67 .verify_checksum = options.verify_checksum,84 const decode_state = decompress.block.DecodeState.init(
68 .checksum = undefined,85 &self.literal_fse_buffer,
69 .current_frame_decompressed_size = undefined,86 &self.match_fse_buffer,
70 };87 &self.offset_fse_buffer,
88 );
89
90 self.decode_state = decode_state;
91 self.frame_context = frame_context;
92
93 self.checksum = null;
94 self.current_frame_decompressed_size = 0;
95
96 self.state = .InFrame;
97 },
71 }98 }
99 }
72100
73 fn frameInit(self: *Self) !void {101 pub fn reader(self: *Self) std.io.Reader {
74 const source_reader = self.source.reader();102 return .{ .context = self };
75 switch (try decompress.decodeFrameHeader(source_reader)) {103 }
76 .skippable => |header| {104
77 try source_reader.skipBytes(header.frame_size, .{});105 pub fn read(self: *Self, buffer: []u8) Error!usize {
78 self.state = .NewFrame;106 if (buffer.len == 0) return 0;
79 },107
80 .zstandard => |header| {108 var size: usize = 0;
81 const frame_context = try decompress.FrameContext.init(109 while (size == 0) {
82 header,110 while (self.state == .NewFrame) {
83 self.buffer.data.len,111 const initial_count = self.source.bytes_read;
84 self.verify_checksum,112 self.frameInit() catch |err| switch (err) {
85 );113 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
86114 error.EndOfStream => return if (self.source.bytes_read == initial_count)
87 const decode_state = decompress.block.DecodeState.init(115 0
88 &self.literal_fse_buffer,116 else
89 &self.match_fse_buffer,117 error.MalformedFrame,
90 &self.offset_fse_buffer,118 else => return error.MalformedFrame,
91 );119 };
92
93 self.decode_state = decode_state;
94 self.frame_context = frame_context;
95
96 self.checksum = null;
97 self.current_frame_decompressed_size = 0;
98
99 self.state = .InFrame;
100 },
101 }120 }
121 size = try self.readInner(buffer);
102 }122 }
123 return size;
124 }
103125
104 pub fn reader(self: *Self) Reader {126 fn readInner(self: *Self, buffer: []u8) Error!usize {
105 return .{ .context = self };127 std.debug.assert(self.state != .NewFrame);
106 }
107128
108 pub fn read(self: *Self, buffer: []u8) Error!usize {129 var ring_buffer = RingBuffer{
109 if (buffer.len == 0) return 0;130 .data = self.buffer.data,
110131 .read_index = self.buffer.read_index,
111 var size: usize = 0;132 .write_index = self.buffer.write_index,
112 while (size == 0) {133 };
113 while (self.state == .NewFrame) {134 defer {
114 const initial_count = self.source.bytes_read;135 self.buffer.read_index = ring_buffer.read_index;
115 self.frameInit() catch |err| switch (err) {136 self.buffer.write_index = ring_buffer.write_index;
116 error.DictionaryIdFlagUnsupported => return error.DictionaryIdFlagUnsupported,
117 error.EndOfStream => return if (self.source.bytes_read == initial_count)
118 0
119 else
120 error.MalformedFrame,
121 else => return error.MalformedFrame,
122 };
123 }
124 size = try self.readInner(buffer);
125 }
126 return size;
127 }137 }
128138
129 fn readInner(self: *Self, buffer: []u8) Error!usize {139 const source_reader = self.source;
130 std.debug.assert(self.state != .NewFrame);140 while (ring_buffer.isEmpty() and self.state != .LastBlock) {
131141 const header_bytes = source_reader.readBytesNoEof(3) catch
132 var ring_buffer = RingBuffer{142 return error.MalformedFrame;
133 .data = self.buffer.data,143 const block_header = decompress.block.decodeBlockHeader(&header_bytes);
134 .read_index = self.buffer.read_index,144
135 .write_index = self.buffer.write_index,145 decompress.block.decodeBlockReader(
136 };146 &ring_buffer,
137 defer {147 source_reader,
138 self.buffer.read_index = ring_buffer.read_index;148 block_header,
139 self.buffer.write_index = ring_buffer.write_index;149 &self.decode_state,
150 self.frame_context.block_size_max,
151 &self.literals_buffer,
152 &self.sequence_buffer,
153 ) catch
154 return error.MalformedBlock;
155
156 if (self.frame_context.content_size) |size| {
157 if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
140 }158 }
141159
142 const source_reader = self.source.reader();160 const size = ring_buffer.len();
143 while (ring_buffer.isEmpty() and self.state != .LastBlock) {161 self.current_frame_decompressed_size += size;
144 const header_bytes = source_reader.readBytesNoEof(3) catch
145 return error.MalformedFrame;
146 const block_header = decompress.block.decodeBlockHeader(&header_bytes);
147
148 decompress.block.decodeBlockReader(
149 &ring_buffer,
150 source_reader,
151 block_header,
152 &self.decode_state,
153 self.frame_context.block_size_max,
154 &self.literals_buffer,
155 &self.sequence_buffer,
156 ) catch
157 return error.MalformedBlock;
158
159 if (self.frame_context.content_size) |size| {
160 if (self.current_frame_decompressed_size > size) return error.MalformedFrame;
161 }
162162
163 const size = ring_buffer.len();163 if (self.frame_context.hasher_opt) |*hasher| {
164 self.current_frame_decompressed_size += size;164 if (size > 0) {
165165 const written_slice = ring_buffer.sliceLast(size);
166 if (self.frame_context.hasher_opt) |*hasher| {166 hasher.update(written_slice.first);
167 if (size > 0) {167 hasher.update(written_slice.second);
168 const written_slice = ring_buffer.sliceLast(size);
169 hasher.update(written_slice.first);
170 hasher.update(written_slice.second);
171 }
172 }168 }
173 if (block_header.last_block) {169 }
174 self.state = .LastBlock;170 if (block_header.last_block) {
175 if (self.frame_context.has_checksum) {171 self.state = .LastBlock;
176 const checksum = source_reader.readInt(u32, .little) catch172 if (self.frame_context.has_checksum) {
177 return error.MalformedFrame;173 const checksum = source_reader.readInt(u32, .little) catch
178 if (self.verify_checksum) {174 return error.MalformedFrame;
179 if (self.frame_context.hasher_opt) |*hasher| {175 if (self.verify_checksum) {
180 if (checksum != decompress.computeChecksum(hasher))176 if (self.frame_context.hasher_opt) |*hasher| {
181 return error.ChecksumFailure;177 if (checksum != decompress.computeChecksum(hasher))
182 }178 return error.ChecksumFailure;
183 }179 }
184 }180 }
185 if (self.frame_context.content_size) |content_size| {181 }
186 if (content_size != self.current_frame_decompressed_size) {182 if (self.frame_context.content_size) |content_size| {
187 return error.MalformedFrame;183 if (content_size != self.current_frame_decompressed_size) {
188 }184 return error.MalformedFrame;
189 }185 }
190 }186 }
191 }187 }
188 }
192189
193 const size = @min(ring_buffer.len(), buffer.len);190 const size = @min(ring_buffer.len(), buffer.len);
194 if (size > 0) {191 if (size > 0) {
195 ring_buffer.readFirstAssumeLength(buffer, size);192 ring_buffer.readFirstAssumeLength(buffer, size);
196 }
197 if (self.state == .LastBlock and ring_buffer.len() == 0) {
198 self.state = .NewFrame;
199 }
200 return size;
201 }193 }
202 };194 if (self.state == .LastBlock and ring_buffer.len() == 0) {
203}195 self.state = .NewFrame;
196 }
197 return size;
198 }
199};
204200
205pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {201pub fn decompressor(reader: anytype, options: DecompressorOptions) Decompressor(@TypeOf(reader)) {
206 return Decompressor(@TypeOf(reader)).init(reader, options);202 return Decompressor(@TypeOf(reader)).init(reader, options);
lib/std/debug/Dwarf.zig+17-3
...@@ -2212,7 +2212,7 @@ pub const ElfModule = struct {...@@ -2212,7 +2212,7 @@ pub const ElfModule = struct {
2212 var separate_debug_filename: ?[]const u8 = null;2212 var separate_debug_filename: ?[]const u8 = null;
2213 var separate_debug_crc: ?u32 = null;2213 var separate_debug_crc: ?u32 = null;
22142214
2215 for (shdrs) |*shdr| {2215 shdrs: for (shdrs) |*shdr| {
2216 if (shdr.sh_type == elf.SHT_NULL or shdr.sh_type == elf.SHT_NOBITS) continue;2216 if (shdr.sh_type == elf.SHT_NULL or shdr.sh_type == elf.SHT_NOBITS) continue;
2217 const name = mem.sliceTo(header_strings[shdr.sh_name..], 0);2217 const name = mem.sliceTo(header_strings[shdr.sh_name..], 0);
22182218
...@@ -2246,8 +2246,22 @@ pub const ElfModule = struct {...@@ -2246,8 +2246,22 @@ pub const ElfModule = struct {
2246 const decompressed_section = try gpa.alloc(u8, ch_size);2246 const decompressed_section = try gpa.alloc(u8, ch_size);
2247 errdefer gpa.free(decompressed_section);2247 errdefer gpa.free(decompressed_section);
22482248
2249 const read = zlib_stream.reader().readAll(decompressed_section) catch continue;2249 {
2250 assert(read == decompressed_section.len);2250 var read_index: usize = 0;
2251 while (true) {
2252 const read_result = zlib_stream.streamReadVec(&.{decompressed_section[read_index..]});
2253 read_result.err catch {
2254 gpa.free(decompressed_section);
2255 continue :shdrs;
2256 };
2257 read_index += read_result.len;
2258 if (read_index == decompressed_section.len) break;
2259 if (read_result.end) {
2260 gpa.free(decompressed_section);
2261 continue :shdrs;
2262 }
2263 }
2264 }
22512265
2252 break :blk .{2266 break :blk .{
2253 .data = decompressed_section,2267 .data = decompressed_section,
lib/std/debug/FixedBufferReader.zig+21-9
...@@ -1,5 +1,7 @@...@@ -1,5 +1,7 @@
1//! Optimized for performance in debug builds.1//! Optimized for performance in debug builds.
22
3// TODO I'm pretty sure this can be deleted thanks to the new std.io.BufferedReader semantics
4
3const std = @import("../std.zig");5const std = @import("../std.zig");
4const MemoryAccessor = std.debug.MemoryAccessor;6const MemoryAccessor = std.debug.MemoryAccessor;
57
...@@ -9,20 +11,20 @@ buf: []const u8,...@@ -9,20 +11,20 @@ buf: []const u8,
9pos: usize = 0,11pos: usize = 0,
10endian: std.builtin.Endian,12endian: std.builtin.Endian,
1113
12pub const Error = error{ EndOfBuffer, Overflow, InvalidBuffer };14pub const Error = error{ EndOfStream, Overflow, InvalidBuffer };
1315
14pub fn seekTo(fbr: *FixedBufferReader, pos: u64) Error!void {16pub fn seekTo(fbr: *FixedBufferReader, pos: u64) Error!void {
15 if (pos > fbr.buf.len) return error.EndOfBuffer;17 if (pos > fbr.buf.len) return error.EndOfStream;
16 fbr.pos = @intCast(pos);18 fbr.pos = @intCast(pos);
17}19}
1820
19pub fn seekForward(fbr: *FixedBufferReader, amount: u64) Error!void {21pub fn seekForward(fbr: *FixedBufferReader, amount: u64) Error!void {
20 if (fbr.buf.len - fbr.pos < amount) return error.EndOfBuffer;22 if (fbr.buf.len - fbr.pos < amount) return error.EndOfStream;
21 fbr.pos += @intCast(amount);23 fbr.pos += @intCast(amount);
22}24}
2325
24pub inline fn readByte(fbr: *FixedBufferReader) Error!u8 {26pub inline fn readByte(fbr: *FixedBufferReader) Error!u8 {
25 if (fbr.pos >= fbr.buf.len) return error.EndOfBuffer;27 if (fbr.pos >= fbr.buf.len) return error.EndOfStream;
26 defer fbr.pos += 1;28 defer fbr.pos += 1;
27 return fbr.buf[fbr.pos];29 return fbr.buf[fbr.pos];
28}30}
...@@ -33,7 +35,7 @@ pub fn readByteSigned(fbr: *FixedBufferReader) Error!i8 {...@@ -33,7 +35,7 @@ pub fn readByteSigned(fbr: *FixedBufferReader) Error!i8 {
3335
34pub fn readInt(fbr: *FixedBufferReader, comptime T: type) Error!T {36pub fn readInt(fbr: *FixedBufferReader, comptime T: type) Error!T {
35 const size = @divExact(@typeInfo(T).int.bits, 8);37 const size = @divExact(@typeInfo(T).int.bits, 8);
36 if (fbr.buf.len - fbr.pos < size) return error.EndOfBuffer;38 if (fbr.buf.len - fbr.pos < size) return error.EndOfStream;
37 defer fbr.pos += size;39 defer fbr.pos += size;
38 return std.mem.readInt(T, fbr.buf[fbr.pos..][0..size], fbr.endian);40 return std.mem.readInt(T, fbr.buf[fbr.pos..][0..size], fbr.endian);
39}41}
...@@ -50,11 +52,21 @@ pub fn readIntChecked(...@@ -50,11 +52,21 @@ pub fn readIntChecked(
50}52}
5153
52pub fn readUleb128(fbr: *FixedBufferReader, comptime T: type) Error!T {54pub fn readUleb128(fbr: *FixedBufferReader, comptime T: type) Error!T {
53 return std.leb.readUleb128(T, fbr);55 var br: std.io.BufferedReader = undefined;
56 br.initFixed(fbr.buf);
57 br.seek = fbr.pos;
58 const result = br.takeUleb128(T);
59 fbr.pos = br.seek;
60 return @errorCast(result);
54}61}
5562
56pub fn readIleb128(fbr: *FixedBufferReader, comptime T: type) Error!T {63pub fn readIleb128(fbr: *FixedBufferReader, comptime T: type) Error!T {
57 return std.leb.readIleb128(T, fbr);64 var br: std.io.BufferedReader = undefined;
65 br.initFixed(fbr.buf);
66 br.seek = fbr.pos;
67 const result = br.takeIleb128(T);
68 fbr.pos = br.seek;
69 return @errorCast(result);
58}70}
5971
60pub fn readAddress(fbr: *FixedBufferReader, format: std.dwarf.Format) Error!u64 {72pub fn readAddress(fbr: *FixedBufferReader, format: std.dwarf.Format) Error!u64 {
...@@ -76,7 +88,7 @@ pub fn readAddressChecked(...@@ -76,7 +88,7 @@ pub fn readAddressChecked(
76}88}
7789
78pub fn readBytes(fbr: *FixedBufferReader, len: usize) Error![]const u8 {90pub fn readBytes(fbr: *FixedBufferReader, len: usize) Error![]const u8 {
79 if (fbr.buf.len - fbr.pos < len) return error.EndOfBuffer;91 if (fbr.buf.len - fbr.pos < len) return error.EndOfStream;
80 defer fbr.pos += len;92 defer fbr.pos += len;
81 return fbr.buf[fbr.pos..][0..len];93 return fbr.buf[fbr.pos..][0..len];
82}94}
...@@ -87,7 +99,7 @@ pub fn readBytesTo(fbr: *FixedBufferReader, comptime sentinel: u8) Error![:senti...@@ -87,7 +99,7 @@ pub fn readBytesTo(fbr: *FixedBufferReader, comptime sentinel: u8) Error![:senti
87 fbr.buf,99 fbr.buf,
88 fbr.pos,100 fbr.pos,
89 sentinel,101 sentinel,
90 }) orelse return error.EndOfBuffer;102 }) orelse return error.EndOfStream;
91 defer fbr.pos = end + 1;103 defer fbr.pos = end + 1;
92 return fbr.buf[fbr.pos..end :sentinel];104 return fbr.buf[fbr.pos..end :sentinel];
93}105}
lib/std/debug/SelfInfo.zig+4-4
...@@ -2028,13 +2028,13 @@ pub const VirtualMachine = struct {...@@ -2028,13 +2028,13 @@ pub const VirtualMachine = struct {
2028 var prev_row: Row = self.current_row;2028 var prev_row: Row = self.current_row;
20292029
2030 var cie_stream: std.io.BufferedReader = undefined;2030 var cie_stream: std.io.BufferedReader = undefined;
2031 cie_stream.initFixed(&cie.initial_instructions);2031 cie_stream.initFixed(cie.initial_instructions);
2032 var fde_stream: std.io.BufferedReader = undefined;2032 var fde_stream: std.io.BufferedReader = undefined;
2033 fde_stream.initFixed(&fde.instructions);2033 fde_stream.initFixed(fde.instructions);
2034 const streams: [2]*std.io.FixedBufferStream = .{ &cie_stream, &fde_stream };2034 const streams: [2]*std.io.BufferedReader = .{ &cie_stream, &fde_stream };
20352035
2036 for (&streams, 0..) |stream, i| {2036 for (&streams, 0..) |stream, i| {
2037 while (stream.pos < stream.buffer.len) {2037 while (stream.seek < stream.buffer.len) {
2038 const instruction = try std.debug.Dwarf.call_frame.Instruction.read(stream, addr_size_bytes, endian);2038 const instruction = try std.debug.Dwarf.call_frame.Instruction.read(stream, addr_size_bytes, endian);
2039 prev_row = try self.step(allocator, cie, i == 0, instruction);2039 prev_row = try self.step(allocator, cie, i == 0, instruction);
2040 if (pc < fde.pc_begin + self.current_row.offset) return prev_row;2040 if (pc < fde.pc_begin + self.current_row.offset) return prev_row;
lib/std/fmt.zig+10-8
...@@ -91,7 +91,7 @@ pub const Options = struct {...@@ -91,7 +91,7 @@ pub const Options = struct {
91/// A user type may be a `struct`, `vector`, `union` or `enum` type.91/// A user type may be a `struct`, `vector`, `union` or `enum` type.
92///92///
93/// To print literal curly braces, escape them by writing them twice, e.g. `{{` or `}}`.93/// To print literal curly braces, escape them by writing them twice, e.g. `{{` or `}}`.
94pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytype) anyerror!void {94pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytype) anyerror!usize {
95 const ArgsType = @TypeOf(args);95 const ArgsType = @TypeOf(args);
96 const args_type_info = @typeInfo(ArgsType);96 const args_type_info = @typeInfo(ArgsType);
97 if (args_type_info != .@"struct") {97 if (args_type_info != .@"struct") {
...@@ -107,6 +107,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp...@@ -107,6 +107,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp
107 comptime var arg_state: ArgState = .{ .args_len = fields_info.len };107 comptime var arg_state: ArgState = .{ .args_len = fields_info.len };
108 comptime var i = 0;108 comptime var i = 0;
109 comptime var literal: []const u8 = "";109 comptime var literal: []const u8 = "";
110 var bytes_written: usize = 0;
110 inline while (true) {111 inline while (true) {
111 const start_index = i;112 const start_index = i;
112113
...@@ -136,7 +137,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp...@@ -136,7 +137,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp
136137
137 // Write out the literal138 // Write out the literal
138 if (literal.len != 0) {139 if (literal.len != 0) {
139 try bw.writeAll(literal);140 bytes_written += try bw.writeAllCount(literal);
140 literal = "";141 literal = "";
141 }142 }
142143
...@@ -196,7 +197,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp...@@ -196,7 +197,7 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp
196 const arg_to_print = comptime arg_state.nextArg(arg_pos) orelse197 const arg_to_print = comptime arg_state.nextArg(arg_pos) orelse
197 @compileError("too few arguments");198 @compileError("too few arguments");
198199
199 try bw.printValue(200 bytes_written += try bw.printValue(
200 placeholder.specifier_arg,201 placeholder.specifier_arg,
201 .{202 .{
202 .fill = placeholder.fill,203 .fill = placeholder.fill,
...@@ -217,6 +218,8 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp...@@ -217,6 +218,8 @@ pub fn format(bw: *std.io.BufferedWriter, comptime fmt: []const u8, args: anytyp
217 else => @compileError(comptimePrint("{d}", .{missing_count}) ++ " unused arguments in '" ++ fmt ++ "'"),218 else => @compileError(comptimePrint("{d}", .{missing_count}) ++ " unused arguments in '" ++ fmt ++ "'"),
218 }219 }
219 }220 }
221
222 return bytes_written;
220}223}
221224
222fn cacheString(str: anytype) []const u8 {225fn cacheString(str: anytype) []const u8 {
...@@ -852,11 +855,10 @@ pub fn bufPrintZ(buf: []u8, comptime fmt: []const u8, args: anytype) BufPrintErr...@@ -852,11 +855,10 @@ pub fn bufPrintZ(buf: []u8, comptime fmt: []const u8, args: anytype) BufPrintErr
852}855}
853856
854/// Count the characters needed for format.857/// Count the characters needed for format.
855pub fn count(comptime fmt: []const u8, args: anytype) u64 {858pub fn count(comptime fmt: []const u8, args: anytype) usize {
856 var counting_writer: std.io.CountingWriter = .{ .child_writer = std.io.null_writer };859 var buffer: [std.atomic.cache_line]u8 = undefined;
857 var bw = counting_writer.writer().unbuffered();860 var bw = std.io.Writer.null.buffered(&buffer);
858 bw.print(fmt, args) catch unreachable;861 return bw.printCount(fmt, args) catch unreachable;
859 return counting_writer.bytes_written;
860}862}
861863
862pub const AllocPrintError = error{OutOfMemory};864pub const AllocPrintError = error{OutOfMemory};
lib/std/fs/File.zig+43-18
...@@ -1512,23 +1512,48 @@ pub fn writeFileAllUnseekable(self: File, in_file: File, args: WriteFileOptions)...@@ -1512,23 +1512,48 @@ pub fn writeFileAllUnseekable(self: File, in_file: File, args: WriteFileOptions)
1512 return @errorCast(writeFileAllUnseekableInner(self, in_file, args));1512 return @errorCast(writeFileAllUnseekableInner(self, in_file, args));
1513}1513}
15141514
1515fn writeFileAllUnseekableInner(self: File, in_file: File, args: WriteFileOptions) anyerror!void {1515fn writeFileAllUnseekableInner(out_file: File, in_file: File, args: WriteFileOptions) anyerror!void {
1516 const headers = args.headers_and_trailers[0..args.header_count];1516 const headers = args.headers_and_trailers[0..args.header_count];
1517 const trailers = args.headers_and_trailers[args.header_count..];1517 const trailers = args.headers_and_trailers[args.header_count..];
15181518
1519 try self.writevAll(headers);1519 try out_file.writevAll(headers);
15201520
1521 try in_file.reader().skipBytes(args.in_offset, .{ .buf_size = 4096 });1521 // Some possible optimizations here:
1522 // * Could writev buffer multiple times if the amount to discard is larger than 4096
1523 // * Could combine discard and read in one readv if amount to discard is small
15221524
1523 var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();1525 var buffer: [4096]u8 = undefined;
1526 var remaining = args.in_offset;
1527 while (remaining > 0) {
1528 const n = try in_file.read(buffer[0..@min(buffer.len, remaining)]);
1529 if (n == 0) return error.EndOfStream;
1530 remaining -= n;
1531 }
1524 if (args.in_len) |len| {1532 if (args.in_len) |len| {
1525 var stream = std.io.limitedReader(in_file.reader(), len);1533 remaining = len;
1526 try fifo.pump(stream.reader(), self.writer());1534 var buffer_index: usize = 0;
1535 while (remaining > 0) {
1536 const n = buffer_index + try in_file.read(buffer[buffer_index..@min(buffer.len, remaining)]);
1537 if (n == 0) return error.EndOfStream;
1538 const written = try out_file.write(buffer[0..n]);
1539 if (written == 0) return error.EndOfStream;
1540 remaining -= written;
1541 std.mem.copyForwards(u8, &buffer, buffer[written..n]);
1542 buffer_index = n - written;
1543 }
1527 } else {1544 } else {
1528 try fifo.pump(in_file.reader(), self.writer());1545 var buffer_index: usize = 0;
1546 while (true) {
1547 const n = buffer_index + try in_file.read(buffer[buffer_index..]);
1548 if (n == 0) break;
1549 const written = try out_file.write(buffer[0..n]);
1550 if (written == 0) return error.EndOfStream;
1551 std.mem.copyForwards(u8, &buffer, buffer[written..n]);
1552 buffer_index = n - written;
1553 }
1529 }1554 }
15301555
1531 try self.writevAll(trailers);1556 try out_file.writevAll(trailers);
1532}1557}
15331558
1534/// Low level function which can fail for OS-specific reasons.1559/// Low level function which can fail for OS-specific reasons.
...@@ -1645,7 +1670,7 @@ pub fn reader_posReadVec(context: *anyopaque, data: []const []u8, offset: u64) a...@@ -1645,7 +1670,7 @@ pub fn reader_posReadVec(context: *anyopaque, data: []const []u8, offset: u64) a
1645}1670}
16461671
1647pub fn reader_streamRead(1672pub fn reader_streamRead(
1648 context: *anyopaque,1673 context: ?*anyopaque,
1649 bw: *std.io.BufferedWriter,1674 bw: *std.io.BufferedWriter,
1650 limit: std.io.Reader.Limit,1675 limit: std.io.Reader.Limit,
1651) anyerror!std.io.Reader.Status {1676) anyerror!std.io.Reader.Status {
...@@ -1658,7 +1683,7 @@ pub fn reader_streamRead(...@@ -1658,7 +1683,7 @@ pub fn reader_streamRead(
1658 };1683 };
1659}1684}
16601685
1661pub fn reader_streamReadVec(context: *anyopaque, data: []const []u8) anyerror!std.io.Reader.Status {1686pub fn reader_streamReadVec(context: ?*anyopaque, data: []const []u8) anyerror!std.io.Reader.Status {
1662 const file = opaqueToHandle(context);1687 const file = opaqueToHandle(context);
1663 const n = try file.readv(data);1688 const n = try file.readv(data);
1664 return .{1689 return .{
...@@ -1667,12 +1692,12 @@ pub fn reader_streamReadVec(context: *anyopaque, data: []const []u8) anyerror!st...@@ -1667,12 +1692,12 @@ pub fn reader_streamReadVec(context: *anyopaque, data: []const []u8) anyerror!st
1667 };1692 };
1668}1693}
16691694
1670pub fn writer_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {1695pub fn writer_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Result {
1671 const file = opaqueToHandle(context);1696 const file = opaqueToHandle(context);
1672 var splat_buffer: [256]u8 = undefined;1697 var splat_buffer: [256]u8 = undefined;
1673 if (is_windows) {1698 if (is_windows) {
1674 if (data.len == 1 and splat == 0) return 0;1699 if (data.len == 1 and splat == 0) return 0;
1675 return windows.WriteFile(file, data[0], null);1700 return .{ .len = windows.WriteFile(file, data[0], null) catch |err| return .{ .err = err } };
1676 }1701 }
1677 var iovecs: [max_buffers_len]std.posix.iovec_const = undefined;1702 var iovecs: [max_buffers_len]std.posix.iovec_const = undefined;
1678 var len: usize = @min(iovecs.len, data.len);1703 var len: usize = @min(iovecs.len, data.len);
...@@ -1681,8 +1706,8 @@ pub fn writer_writeSplat(context: *anyopaque, data: []const []const u8, splat: u...@@ -1681,8 +1706,8 @@ pub fn writer_writeSplat(context: *anyopaque, data: []const []const u8, splat: u
1681 .len = d.len,1706 .len = d.len,
1682 };1707 };
1683 switch (splat) {1708 switch (splat) {
1684 0 => return std.posix.writev(file, iovecs[0 .. len - 1]),1709 0 => return .{ .len = std.posix.writev(file, iovecs[0 .. len - 1]) catch |err| return .{ .err = err } },
1685 1 => return std.posix.writev(file, iovecs[0..len]),1710 1 => return .{ .len = std.posix.writev(file, iovecs[0..len]) catch |err| return .{ .err = err } },
1686 else => {1711 else => {
1687 const pattern = data[data.len - 1];1712 const pattern = data[data.len - 1];
1688 if (pattern.len == 1) {1713 if (pattern.len == 1) {
...@@ -1700,21 +1725,21 @@ pub fn writer_writeSplat(context: *anyopaque, data: []const []const u8, splat: u...@@ -1700,21 +1725,21 @@ pub fn writer_writeSplat(context: *anyopaque, data: []const []const u8, splat: u
1700 iovecs[len] = .{ .base = &splat_buffer, .len = remaining_splat };1725 iovecs[len] = .{ .base = &splat_buffer, .len = remaining_splat };
1701 len += 1;1726 len += 1;
1702 }1727 }
1703 return std.posix.writev(file, iovecs[0..len]);1728 return .{ .len = std.posix.writev(file, iovecs[0..len]) catch |err| return .{ .err = err } };
1704 }1729 }
1705 },1730 },
1706 }1731 }
1707 return std.posix.writev(file, iovecs[0..len]);1732 return .{ .len = std.posix.writev(file, iovecs[0..len]) catch |err| return .{ .err = err } };
1708}1733}
17091734
1710pub fn writer_writeFile(1735pub fn writer_writeFile(
1711 context: *anyopaque,1736 context: ?*anyopaque,
1712 in_file: std.fs.File,1737 in_file: std.fs.File,
1713 in_offset: u64,1738 in_offset: u64,
1714 in_len: std.io.Writer.FileLen,1739 in_len: std.io.Writer.FileLen,
1715 headers_and_trailers: []const []const u8,1740 headers_and_trailers: []const []const u8,
1716 headers_len: usize,1741 headers_len: usize,
1717) anyerror!usize {1742) std.io.Writer.Result {
1718 const out_fd = opaqueToHandle(context);1743 const out_fd = opaqueToHandle(context);
1719 const in_fd = in_file.handle;1744 const in_fd = in_file.handle;
1720 const len_int = switch (in_len) {1745 const len_int = switch (in_len) {
lib/std/io.zig-44
...@@ -20,8 +20,6 @@ pub const Writer = @import("io/Writer.zig");...@@ -20,8 +20,6 @@ pub const Writer = @import("io/Writer.zig");
20pub const BufferedReader = @import("io/BufferedReader.zig");20pub const BufferedReader = @import("io/BufferedReader.zig");
21pub const BufferedWriter = @import("io/BufferedWriter.zig");21pub const BufferedWriter = @import("io/BufferedWriter.zig");
22pub const AllocatingWriter = @import("io/AllocatingWriter.zig");22pub const AllocatingWriter = @import("io/AllocatingWriter.zig");
23pub const CountingWriter = @import("io/CountingWriter.zig");
24pub const CountingReader = @import("io/CountingReader.zig");
2523
26pub const CWriter = @import("io/c_writer.zig").CWriter;24pub const CWriter = @import("io/c_writer.zig").CWriter;
27pub const cWriter = @import("io/c_writer.zig").cWriter;25pub const cWriter = @import("io/c_writer.zig").cWriter;
...@@ -48,46 +46,6 @@ pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAt...@@ -48,46 +46,6 @@ pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAt
4846
49pub const tty = @import("io/tty.zig");47pub const tty = @import("io/tty.zig");
5048
51/// A `Writer` that discards all data.
52pub const null_writer: Writer = .{
53 .context = undefined,
54 .vtable = &.{
55 .writeSplat = null_writeSplat,
56 .writeFile = null_writeFile,
57 },
58};
59
60fn null_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {
61 _ = context;
62 const headers = data[0 .. data.len - 1];
63 const pattern = data[headers.len..];
64 var written: usize = pattern.len * splat;
65 for (headers) |bytes| written += bytes.len;
66 return written;
67}
68
69fn null_writeFile(
70 context: *anyopaque,
71 file: std.fs.File,
72 offset: u64,
73 len: Writer.FileLen,
74 headers_and_trailers: []const []const u8,
75 headers_len: usize,
76) anyerror!usize {
77 _ = context;
78 _ = offset;
79 _ = headers_len;
80 _ = file;
81 if (len == .entire_file) return error.Unimplemented;
82 var n: usize = 0;
83 for (headers_and_trailers) |bytes| n += bytes.len;
84 return len.int() + n;
85}
86
87test null_writer {
88 try null_writer.writeAll("yay");
89}
90
91pub fn poll(49pub fn poll(
92 allocator: Allocator,50 allocator: Allocator,
93 comptime StreamEnum: type,51 comptime StreamEnum: type,
...@@ -494,8 +452,6 @@ test {...@@ -494,8 +452,6 @@ test {
494 _ = BufferedReader;452 _ = BufferedReader;
495 _ = Reader;453 _ = Reader;
496 _ = Writer;454 _ = Writer;
497 _ = CountingWriter;
498 _ = CountingReader;
499 _ = AllocatingWriter;455 _ = AllocatingWriter;
500 _ = @import("io/bit_reader.zig");456 _ = @import("io/bit_reader.zig");
501 _ = @import("io/bit_writer.zig");457 _ = @import("io/bit_writer.zig");
lib/std/io/BufferedReader.zig+162-25
...@@ -14,26 +14,37 @@ seek: usize,...@@ -14,26 +14,37 @@ seek: usize,
14storage: BufferedWriter,14storage: BufferedWriter,
15unbuffered_reader: Reader,15unbuffered_reader: Reader,
1616
17pub fn init(br: *BufferedReader, r: Reader, buffer: []u8) void {
18 br.* = .{
19 .seek = 0,
20 .storage = undefined,
21 .unbuffered_reader = r,
22 };
23 br.storage.initFixed(buffer);
24}
25
26/// Constructs `br` such that it will read from `buffer` and then end.
17pub fn initFixed(br: *BufferedReader, buffer: []const u8) void {27pub fn initFixed(br: *BufferedReader, buffer: []const u8) void {
18 br.* = .{28 br.* = .{
19 .seek = 0,29 .seek = 0,
20 .storage = .{30 .storage = .{
21 .buffer = buffer,31 .buffer = .initBuffer(@constCast(buffer)),
22 .mode = .fixed,32 .unbuffered_writer = .{
23 },33 .context = undefined,
24 .reader = .{34 .vtable = &std.io.Writer.VTable.eof,
25 .context = br,
26 .vtable = &.{
27 .streamRead = null,
28 .posRead = null,
29 },35 },
30 },36 },
37 .unbuffered_reader = &.{
38 .context = undefined,
39 .vtable = &std.io.Reader.VTable.eof,
40 },
31 };41 };
32}42}
3343
34pub fn deinit(br: *BufferedReader) void {44pub fn storageBuffer(br: *BufferedReader) []u8 {
35 br.storage.deinit();45 assert(br.storage.unbuffered_writer.vtable == &std.io.Writer.VTable.eof);
36 br.* = undefined;46 assert(br.unbuffered_reader.vtable == &std.io.Reader.VTable.eof);
47 return br.storage.buffer.allocatedSlice();
37}48}
3849
39/// Although `BufferedReader` can easily satisfy the `Reader` interface, it's50/// Although `BufferedReader` can easily satisfy the `Reader` interface, it's
...@@ -51,30 +62,31 @@ pub fn reader(br: *BufferedReader) Reader {...@@ -51,30 +62,31 @@ pub fn reader(br: *BufferedReader) Reader {
51 };62 };
52}63}
5364
54fn passthru_streamRead(ctx: *anyopaque, bw: *BufferedWriter, limit: Reader.Limit) anyerror!Reader.Status {65fn passthru_streamRead(ctx: ?*anyopaque, bw: *BufferedWriter, limit: Reader.Limit) anyerror!Reader.RwResult {
55 const br: *BufferedReader = @alignCast(@ptrCast(ctx));66 const br: *BufferedReader = @alignCast(@ptrCast(ctx));
56 const buffer = br.storage.buffer.items;67 const buffer = br.storage.buffer.items;
57 const buffered = buffer[br.seek..];68 const buffered = buffer[br.seek..];
58 const limited = buffered[0..limit.min(buffered.len)];69 const limited = buffered[0..limit.min(buffered.len)];
59 if (limited.len > 0) {70 if (limited.len > 0) {
60 const n = try bw.writeSplat(limited, 1);71 const result = bw.writeSplat(limited, 1);
61 br.seek += n;72 br.seek += result.len;
62 return .{73 return .{
63 .end = false,74 .len = result.len,
64 .len = @intCast(n),75 .write_err = result.err,
76 .write_end = result.end,
65 };77 };
66 }78 }
67 return br.unbuffered_reader.streamRead(bw, limit);79 return br.unbuffered_reader.streamRead(bw, limit);
68}80}
6981
70fn passthru_streamReadVec(ctx: *anyopaque, data: []const []u8) anyerror!Reader.Status {82fn passthru_streamReadVec(ctx: ?*anyopaque, data: []const []u8) anyerror!Reader.Status {
71 const br: *BufferedReader = @alignCast(@ptrCast(ctx));83 const br: *BufferedReader = @alignCast(@ptrCast(ctx));
72 _ = br;84 _ = br;
73 _ = data;85 _ = data;
74 @panic("TODO");86 @panic("TODO");
75}87}
7688
77fn passthru_posRead(ctx: *anyopaque, bw: *BufferedWriter, limit: Reader.Limit, off: u64) anyerror!Reader.Status {89fn passthru_posRead(ctx: ?*anyopaque, bw: *BufferedWriter, limit: Reader.Limit, off: u64) anyerror!Reader.Status {
78 const br: *BufferedReader = @alignCast(@ptrCast(ctx));90 const br: *BufferedReader = @alignCast(@ptrCast(ctx));
79 const buffer = br.storage.buffer.items;91 const buffer = br.storage.buffer.items;
80 if (off < buffer.len) {92 if (off < buffer.len) {
...@@ -84,7 +96,7 @@ fn passthru_posRead(ctx: *anyopaque, bw: *BufferedWriter, limit: Reader.Limit, o...@@ -84,7 +96,7 @@ fn passthru_posRead(ctx: *anyopaque, bw: *BufferedWriter, limit: Reader.Limit, o
84 return br.unbuffered_reader.posRead(bw, limit, off - buffer.len);96 return br.unbuffered_reader.posRead(bw, limit, off - buffer.len);
85}97}
8698
87fn passthru_posReadVec(ctx: *anyopaque, data: []const []u8, off: u64) anyerror!Reader.Status {99fn passthru_posReadVec(ctx: ?*anyopaque, data: []const []u8, off: u64) anyerror!Reader.Status {
88 const br: *BufferedReader = @alignCast(@ptrCast(ctx));100 const br: *BufferedReader = @alignCast(@ptrCast(ctx));
89 _ = br;101 _ = br;
90 _ = data;102 _ = data;
...@@ -155,8 +167,24 @@ pub fn takeArray(br: *BufferedReader, comptime n: usize) anyerror!*[n]u8 {...@@ -155,8 +167,24 @@ pub fn takeArray(br: *BufferedReader, comptime n: usize) anyerror!*[n]u8 {
155///167///
156/// See also:168/// See also:
157/// * `toss`169/// * `toss`
158/// * `discardAll`170/// * `discardUntilEnd`
171/// * `discardUpTo`
159pub fn discard(br: *BufferedReader, n: usize) anyerror!void {172pub fn discard(br: *BufferedReader, n: usize) anyerror!void {
173 if ((try discardUpTo(br, n)) != n) return error.EndOfStream;
174}
175
176/// Skips the next `n` bytes from the stream, advancing the seek position.
177///
178/// Unlike `toss` which is infallible, in this function `n` can be any amount.
179///
180/// Returns the number of bytes discarded, which is less than `n` if and only
181/// if the stream reached the end.
182///
183/// See also:
184/// * `discard`
185/// * `toss`
186/// * `discardUntilEnd`
187pub fn discardUpTo(br: *BufferedReader, n: usize) anyerror!usize {
160 const list = &br.storage.buffer;188 const list = &br.storage.buffer;
161 var remaining = n;189 var remaining = n;
162 while (remaining > 0) {190 while (remaining > 0) {
...@@ -168,19 +196,22 @@ pub fn discard(br: *BufferedReader, n: usize) anyerror!void {...@@ -168,19 +196,22 @@ pub fn discard(br: *BufferedReader, n: usize) anyerror!void {
168 remaining -= (list.items.len - br.seek);196 remaining -= (list.items.len - br.seek);
169 list.items.len = 0;197 list.items.len = 0;
170 br.seek = 0;198 br.seek = 0;
171 const status = try br.unbuffered_reader.streamRead(&br.storage, .none);199 const result = try br.unbuffered_reader.streamRead(&br.storage, .none);
200 result.write_err catch unreachable;
201 try result.read_err;
202 assert(result.len == list.items.len);
172 if (remaining <= list.items.len) continue;203 if (remaining <= list.items.len) continue;
173 if (status.end) return error.EndOfStream;204 if (result.end) return n - remaining;
174 }205 }
175}206}
176207
177/// Reads the stream until the end, ignoring all the data.208/// Reads the stream until the end, ignoring all the data.
178/// Returns the number of bytes discarded.209/// Returns the number of bytes discarded.
179pub fn discardAll(br: *BufferedReader) anyerror!usize {210pub fn discardUntilEnd(br: *BufferedReader) anyerror!usize {
180 const list = &br.storage.buffer;211 const list = &br.storage.buffer;
181 var total: usize = list.items.len;212 var total: usize = list.items.len;
182 list.items.len = 0;213 list.items.len = 0;
183 total += try br.unbuffered_reader.discardAll();214 total += try br.unbuffered_reader.discardUntilEnd();
184 return total;215 return total;
185}216}
186217
...@@ -224,6 +255,15 @@ pub fn read(br: *BufferedReader, buffer: []u8) anyerror!void {...@@ -224,6 +255,15 @@ pub fn read(br: *BufferedReader, buffer: []u8) anyerror!void {
224 }255 }
225}256}
226257
258/// Returns the number of bytes read. If the number read is smaller than `buffer.len`, it
259/// means the stream reached the end. Reaching the end of a stream is not an error
260/// condition.
261pub fn partialRead(br: *BufferedReader, buffer: []u8) anyerror!usize {
262 _ = br;
263 _ = buffer;
264 @panic("TODO");
265}
266
227/// Returns a slice of the next bytes of buffered data from the stream until267/// Returns a slice of the next bytes of buffered data from the stream until
228/// `delimiter` is found, advancing the seek position.268/// `delimiter` is found, advancing the seek position.
229///269///
...@@ -463,6 +503,95 @@ pub fn takeEnum(br: *BufferedReader, comptime Enum: type, endian: std.builtin.En...@@ -463,6 +503,95 @@ pub fn takeEnum(br: *BufferedReader, comptime Enum: type, endian: std.builtin.En
463 return std.meta.intToEnum(Enum, int);503 return std.meta.intToEnum(Enum, int);
464}504}
465505
506/// Read a single unsigned LEB128 value from the given reader as type T,
507/// or error.Overflow if the value cannot fit.
508pub fn takeUleb128(br: *std.io.BufferedReader, comptime T: type) anyerror!T {
509 const U = if (@typeInfo(T).int.bits < 8) u8 else T;
510 const ShiftT = std.math.Log2Int(U);
511
512 const max_group = (@typeInfo(U).int.bits + 6) / 7;
513
514 var value: U = 0;
515 var group: ShiftT = 0;
516
517 while (group < max_group) : (group += 1) {
518 const byte = try br.takeByte();
519
520 const ov = @shlWithOverflow(@as(U, byte & 0x7f), group * 7);
521 if (ov[1] != 0) return error.Overflow;
522
523 value |= ov[0];
524 if (byte & 0x80 == 0) break;
525 } else {
526 return error.Overflow;
527 }
528
529 // only applies in the case that we extended to u8
530 if (U != T) {
531 if (value > std.math.maxInt(T)) return error.Overflow;
532 }
533
534 return @truncate(value);
535}
536
537/// Read a single signed LEB128 value from the given reader as type T,
538/// or `error.Overflow` if the value cannot fit.
539pub fn takeIleb128(br: *std.io.BufferedReader, comptime T: type) anyerror!T {
540 const S = if (@typeInfo(T).int.bits < 8) i8 else T;
541 const U = std.meta.Int(.unsigned, @typeInfo(S).int.bits);
542 const ShiftU = std.math.Log2Int(U);
543
544 const max_group = (@typeInfo(U).int.bits + 6) / 7;
545
546 var value = @as(U, 0);
547 var group = @as(ShiftU, 0);
548
549 while (group < max_group) : (group += 1) {
550 const byte = try br.takeByte();
551
552 const shift = group * 7;
553 const ov = @shlWithOverflow(@as(U, byte & 0x7f), shift);
554 if (ov[1] != 0) {
555 // Overflow is ok so long as the sign bit is set and this is the last byte
556 if (byte & 0x80 != 0) return error.Overflow;
557 if (@as(S, @bitCast(ov[0])) >= 0) return error.Overflow;
558
559 // and all the overflowed bits are 1
560 const remaining_shift = @as(u3, @intCast(@typeInfo(U).int.bits - @as(u16, shift)));
561 const remaining_bits = @as(i8, @bitCast(byte | 0x80)) >> remaining_shift;
562 if (remaining_bits != -1) return error.Overflow;
563 } else {
564 // If we don't overflow and this is the last byte and the number being decoded
565 // is negative, check that the remaining bits are 1
566 if ((byte & 0x80 == 0) and (@as(S, @bitCast(ov[0])) < 0)) {
567 const remaining_shift = @as(u3, @intCast(@typeInfo(U).int.bits - @as(u16, shift)));
568 const remaining_bits = @as(i8, @bitCast(byte | 0x80)) >> remaining_shift;
569 if (remaining_bits != -1) return error.Overflow;
570 }
571 }
572
573 value |= ov[0];
574 if (byte & 0x80 == 0) {
575 const needs_sign_ext = group + 1 < max_group;
576 if (byte & 0x40 != 0 and needs_sign_ext) {
577 const ones = @as(S, -1);
578 value |= @as(U, @bitCast(ones)) << (shift + 7);
579 }
580 break;
581 }
582 } else {
583 return error.Overflow;
584 }
585
586 const result = @as(S, @bitCast(value));
587 // Only applies if we extended to i8
588 if (S != T) {
589 if (result > std.math.maxInt(T) or result < std.math.minInt(T)) return error.Overflow;
590 }
591
592 return @truncate(result);
593}
594
466test initFixed {595test initFixed {
467 var br: BufferedReader = undefined;596 var br: BufferedReader = undefined;
468 br.initFixed("a\x02");597 br.initFixed("a\x02");
...@@ -501,7 +630,7 @@ test discard {...@@ -501,7 +630,7 @@ test discard {
501 try testing.expectError(error.EndOfStream, br.discard(1));630 try testing.expectError(error.EndOfStream, br.discard(1));
502}631}
503632
504test discardAll {633test discardUntilEnd {
505 return error.Unimplemented;634 return error.Unimplemented;
506}635}
507636
...@@ -576,3 +705,11 @@ test takeStructEndian {...@@ -576,3 +705,11 @@ test takeStructEndian {
576test takeEnum {705test takeEnum {
577 return error.Unimplemented;706 return error.Unimplemented;
578}707}
708
709test takeUleb128 {
710 return error.Unimplemented;
711}
712
713test takeIleb128 {
714 return error.Unimplemented;
715}
lib/std/io/BufferedWriter.zig+334-142
...@@ -43,7 +43,7 @@ const fixed_vtable: Writer.VTable = .{...@@ -43,7 +43,7 @@ const fixed_vtable: Writer.VTable = .{
43};43};
4444
45/// Replaces the `BufferedWriter` with a new one that writes to `buffer` and45/// Replaces the `BufferedWriter` with a new one that writes to `buffer` and
46/// returns `error.NoSpaceLeft` when it is full.46/// then ends when it is full.
47pub fn initFixed(bw: *BufferedWriter, buffer: []u8) void {47pub fn initFixed(bw: *BufferedWriter, buffer: []u8) void {
48 bw.* = .{48 bw.* = .{
49 .unbuffered_writer = .{49 .unbuffered_writer = .{
...@@ -77,6 +77,36 @@ pub fn unusedCapacitySlice(bw: *const BufferedWriter) []u8 {...@@ -77,6 +77,36 @@ pub fn unusedCapacitySlice(bw: *const BufferedWriter) []u8 {
77 return bw.buffer.unusedCapacitySlice();77 return bw.buffer.unusedCapacitySlice();
78}78}
7979
80pub fn writableSlice(bw: *BufferedWriter, minimum_length: usize) anyerror![]u8 {
81 const list = &bw.buffer;
82 assert(list.capacity >= minimum_length);
83 const cap_slice = list.unusedCapacitySlice();
84 if (cap_slice.len >= minimum_length) {
85 @branchHint(.likely);
86 return cap_slice;
87 }
88 const buffer = list.items;
89 const result = bw.unbuffered_writer.write(buffer);
90 if (result.len == buffer.len) {
91 @branchHint(.likely);
92 list.items.len = 0;
93 try result.err;
94 return list.unusedCapacitySlice();
95 }
96 if (result.len > 0) {
97 const remainder = buffer[result.len..];
98 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
99 list.items.len = remainder.len;
100 }
101 try result.err;
102 return list.unusedCapacitySlice();
103}
104
105/// After calling `writableSlice`, this function tracks how many bytes were written to it.
106pub fn advance(bw: *BufferedWriter, n: usize) void {
107 bw.items.len += n;
108}
109
80/// The `data` parameter is mutable because this function needs to mutate the110/// The `data` parameter is mutable because this function needs to mutate the
81/// fields in order to handle partial writes from `Writer.VTable.writev`.111/// fields in order to handle partial writes from `Writer.VTable.writev`.
82pub fn writevAll(bw: *BufferedWriter, data: [][]const u8) anyerror!void {112pub fn writevAll(bw: *BufferedWriter, data: [][]const u8) anyerror!void {
...@@ -92,15 +122,15 @@ pub fn writevAll(bw: *BufferedWriter, data: [][]const u8) anyerror!void {...@@ -92,15 +122,15 @@ pub fn writevAll(bw: *BufferedWriter, data: [][]const u8) anyerror!void {
92 }122 }
93}123}
94124
95pub fn writeSplat(bw: *BufferedWriter, data: []const []const u8, splat: usize) anyerror!usize {125pub fn writeSplat(bw: *BufferedWriter, data: []const []const u8, splat: usize) Writer.Result {
96 return passthru_writeSplat(bw, data, splat);126 return passthru_writeSplat(bw, data, splat);
97}127}
98128
99pub fn writev(bw: *BufferedWriter, data: []const []const u8) anyerror!usize {129pub fn writev(bw: *BufferedWriter, data: []const []const u8) Writer.Result {
100 return passthru_writeSplat(bw, data, 1);130 return passthru_writeSplat(bw, data, 1);
101}131}
102132
103fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {133fn passthru_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Result {
104 const bw: *BufferedWriter = @alignCast(@ptrCast(context));134 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
105 const list = &bw.buffer;135 const list = &bw.buffer;
106 const buffer = list.allocatedSlice();136 const buffer = list.allocatedSlice();
...@@ -126,27 +156,45 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi...@@ -126,27 +156,45 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
126 if (len >= remaining_data.len) {156 if (len >= remaining_data.len) {
127 @branchHint(.likely);157 @branchHint(.likely);
128 // Made it past the headers, so we can enable splatting.158 // Made it past the headers, so we can enable splatting.
129 const n = try bw.unbuffered_writer.writeSplat(send_buffers, splat);159 const result = bw.unbuffered_writer.writeSplat(send_buffers, splat);
160 const n = result.len;
130 if (n < end) {161 if (n < end) {
131 @branchHint(.unlikely);162 @branchHint(.unlikely);
132 const remainder = buffer[n..end];163 const remainder = buffer[n..end];
133 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);164 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
134 list.items.len = remainder.len;165 list.items.len = remainder.len;
135 return end - start_end;166 return .{
167 .err = result.err,
168 .len = end - start_end,
169 .end = result.end,
170 };
136 }171 }
137 list.items.len = 0;172 list.items.len = 0;
138 return n - start_end;173 return .{
174 .err = result.err,
175 .len = n - start_end,
176 .end = result.end,
177 };
139 }178 }
140 const n = try bw.unbuffered_writer.writeSplat(send_buffers, 1);179 const result = try bw.unbuffered_writer.writeSplat(send_buffers, 1);
180 const n = result.len;
141 if (n < end) {181 if (n < end) {
142 @branchHint(.unlikely);182 @branchHint(.unlikely);
143 const remainder = buffer[n..end];183 const remainder = buffer[n..end];
144 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);184 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
145 list.items.len = remainder.len;185 list.items.len = remainder.len;
146 return end - start_end;186 return .{
187 .err = result.err,
188 .len = end - start_end,
189 .end = result.end,
190 };
147 }191 }
148 list.items.len = 0;192 list.items.len = 0;
149 return n - start_end;193 return .{
194 .err = result.err,
195 .len = n - start_end,
196 .end = result.end,
197 };
150 }198 }
151199
152 const pattern = data[data.len - 1];200 const pattern = data[data.len - 1];
...@@ -156,7 +204,7 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi...@@ -156,7 +204,7 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
156 // It was added in the loop above; undo it here.204 // It was added in the loop above; undo it here.
157 end -= pattern.len;205 end -= pattern.len;
158 list.items.len = end;206 list.items.len = end;
159 return end - start_end;207 return .{ .len = end - start_end };
160 }208 }
161209
162 const remaining_splat = splat - 1;210 const remaining_splat = splat - 1;
...@@ -164,7 +212,7 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi...@@ -164,7 +212,7 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
164 switch (pattern.len) {212 switch (pattern.len) {
165 0 => {213 0 => {
166 list.items.len = end;214 list.items.len = end;
167 return end - start_end;215 return .{ .len = end - start_end };
168 },216 },
169 1 => {217 1 => {
170 const new_end = end + remaining_splat;218 const new_end = end + remaining_splat;
...@@ -172,20 +220,29 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi...@@ -172,20 +220,29 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
172 @branchHint(.likely);220 @branchHint(.likely);
173 @memset(buffer[end..new_end], pattern[0]);221 @memset(buffer[end..new_end], pattern[0]);
174 list.items.len = new_end;222 list.items.len = new_end;
175 return new_end - start_end;223 return .{ .len = new_end - start_end };
176 }224 }
177 buffers[0] = buffer[0..end];225 buffers[0] = buffer[0..end];
178 buffers[1] = pattern;226 buffers[1] = pattern;
179 const n = try bw.unbuffered_writer.writeSplat(buffers[0..2], remaining_splat);227 const result = bw.unbuffered_writer.writeSplat(buffers[0..2], remaining_splat);
228 const n = result.len;
180 if (n < end) {229 if (n < end) {
181 @branchHint(.unlikely);230 @branchHint(.unlikely);
182 const remainder = buffer[n..end];231 const remainder = buffer[n..end];
183 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);232 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
184 list.items.len = remainder.len;233 list.items.len = remainder.len;
185 return end - start_end;234 return .{
235 .err = result.err,
236 .len = end - start_end,
237 .end = result.end,
238 };
186 }239 }
187 list.items.len = 0;240 list.items.len = 0;
188 return n - start_end;241 return .{
242 .err = result.err,
243 .len = n - start_end,
244 .end = result.end,
245 };
189 },246 },
190 else => {247 else => {
191 const new_end = end + pattern.len * remaining_splat;248 const new_end = end + pattern.len * remaining_splat;
...@@ -195,46 +252,43 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi...@@ -195,46 +252,43 @@ fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usi
195 @memcpy(buffer[end..][0..pattern.len], pattern);252 @memcpy(buffer[end..][0..pattern.len], pattern);
196 }253 }
197 list.items.len = new_end;254 list.items.len = new_end;
198 return new_end - start_end;255 return .{ .len = new_end - start_end };
199 }256 }
200 buffers[0] = buffer[0..end];257 buffers[0] = buffer[0..end];
201 buffers[1] = pattern;258 buffers[1] = pattern;
202 const n = try bw.unbuffered_writer.writeSplat(buffers[0..2], remaining_splat);259 const result = bw.unbuffered_writer.writeSplat(buffers[0..2], remaining_splat);
260 const n = result.len;
203 if (n < end) {261 if (n < end) {
204 @branchHint(.unlikely);262 @branchHint(.unlikely);
205 const remainder = buffer[n..end];263 const remainder = buffer[n..end];
206 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);264 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
207 list.items.len = remainder.len;265 list.items.len = remainder.len;
208 return end - start_end;266 return .{
267 .err = result.err,
268 .len = end - start_end,
269 .end = result.end,
270 };
209 }271 }
210 list.items.len = 0;272 list.items.len = 0;
211 return n - start_end;273 return .{
274 .err = result.err,
275 .len = n - start_end,
276 .end = result.end,
277 };
212 },278 },
213 }279 }
214}280}
215281
216fn fixed_writev(context: *anyopaque, data: []const []const u8) anyerror!usize {
217 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
218 const list = &bw.buffer;
219 // When this function is called it means the buffer got full, so it's time
220 // to return an error. However, we still need to make sure all of the
221 // available buffer has been used.
222 const first = data[0];
223 const dest = list.unusedCapacitySlice();
224 @memcpy(dest, first[0..dest.len]);
225 list.items.len = list.capacity;
226 return error.NoSpaceLeft;
227}
228
229/// When this function is called it means the buffer got full, so it's time282/// When this function is called it means the buffer got full, so it's time
230/// to return an error. However, we still need to make sure all of the283/// to return an error. However, we still need to make sure all of the
231/// available buffer has been filled.284/// available buffer has been filled.
232fn fixed_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {285fn fixed_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Result {
233 const bw: *BufferedWriter = @alignCast(@ptrCast(context));286 const bw: *BufferedWriter = @alignCast(@ptrCast(context));
234 const list = &bw.buffer;287 const list = &bw.buffer;
288 const start_len = list.items.len;
235 for (data) |bytes| {289 for (data) |bytes| {
236 const dest = list.unusedCapacitySlice();290 const dest = list.unusedCapacitySlice();
237 if (dest.len == 0) return error.NoSpaceLeft;291 if (dest.len == 0) return .{ .len = list.items.len - start_len, .end = true };
238 const len = @min(bytes.len, dest.len);292 const len = @min(bytes.len, dest.len);
239 @memcpy(dest[0..len], bytes[0..len]);293 @memcpy(dest[0..len], bytes[0..len]);
240 list.items.len += len;294 list.items.len += len;
...@@ -247,90 +301,153 @@ fn fixed_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize)...@@ -247,90 +301,153 @@ fn fixed_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize)
247 else => for (0..splat - 1) |i| @memcpy(dest[i * pattern.len ..][0..pattern.len], pattern),301 else => for (0..splat - 1) |i| @memcpy(dest[i * pattern.len ..][0..pattern.len], pattern),
248 }302 }
249 list.items.len = list.capacity;303 list.items.len = list.capacity;
250 return error.NoSpaceLeft;304 return .{ .len = list.items.len - start_len, .end = true };
251}305}
252306
253pub fn write(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {307pub fn write(bw: *BufferedWriter, bytes: []const u8) Writer.Result {
254 const list = &bw.buffer;308 const list = &bw.buffer;
255 const buffer = list.allocatedSlice();309 const buffer = list.allocatedSlice();
256 const end = list.items.len;310 const end = list.items.len;
257 const new_end = end + bytes.len;311 const new_end = end + bytes.len;
258 if (new_end > buffer.len) {312 if (new_end > buffer.len) {
259 var data: [2][]const u8 = .{ buffer[0..end], bytes };313 var data: [2][]const u8 = .{ buffer[0..end], bytes };
260 const n = try bw.unbuffered_writer.writev(&data);314 const result = bw.unbuffered_writer.writev(&data);
315 const n = result.len;
261 if (n < end) {316 if (n < end) {
262 @branchHint(.unlikely);317 @branchHint(.unlikely);
263 const remainder = buffer[n..end];318 const remainder = buffer[n..end];
264 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);319 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
265 list.items.len = remainder.len;320 list.items.len = remainder.len;
266 return 0;321 return .{
322 .err = result.err,
323 .len = 0,
324 .end = result.end,
325 };
267 }326 }
268 list.items.len = 0;327 list.items.len = 0;
269 return n - end;328 return .{
329 .err = result.err,
330 .len = n - end,
331 .end = result.end,
332 };
270 }333 }
271 @memcpy(buffer[end..new_end], bytes);334 @memcpy(buffer[end..new_end], bytes);
272 list.items.len = new_end;335 list.items.len = new_end;
273 return bytes.len;336 return bytes.len;
274}337}
275338
276/// This function is provided by the `Writer`, however it is
277/// duplicated here so that `bw` can be passed to `std.fmt.format` directly,
278/// avoiding one indirect function call.
279pub fn writeAll(bw: *BufferedWriter, bytes: []const u8) anyerror!void {339pub fn writeAll(bw: *BufferedWriter, bytes: []const u8) anyerror!void {
340 if ((try writeUntilEnd(bw, bytes)) != bytes.len) return error.WriteStreamEnd;
341}
342
343pub fn writeAllCount(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
344 try writeAll(bw, bytes);
345 return bytes.len;
346}
347
348/// If the number returned is less than `bytes.len` it indicates end of stream.
349pub fn writeUntilEnd(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
280 var index: usize = 0;350 var index: usize = 0;
281 while (index < bytes.len) index += try write(bw, bytes[index..]);351 while (true) {
352 const result = write(bw, bytes[index..]);
353 try result.err;
354 index += result.len;
355 assert(index <= bytes.len);
356 if (index == bytes.len or result.end) return index;
357 }
282}358}
283359
284pub fn print(bw: *BufferedWriter, comptime format: []const u8, args: anytype) anyerror!void {360pub fn print(bw: *BufferedWriter, comptime format: []const u8, args: anytype) anyerror!void {
361 _ = try std.fmt.format(bw, format, args);
362}
363
364pub fn printCount(bw: *BufferedWriter, comptime format: []const u8, args: anytype) anyerror!usize {
285 return std.fmt.format(bw, format, args);365 return std.fmt.format(bw, format, args);
286}366}
287367
288pub fn writeByte(bw: *BufferedWriter, byte: u8) anyerror!void {368pub fn writeByte(bw: *BufferedWriter, byte: u8) anyerror!void {
369 if ((try writeByteUntilEnd(bw, byte)) == 0) return error.WriteStreamEnd;
370}
371
372pub fn writeByteCount(bw: *BufferedWriter, byte: u8) anyerror!usize {
373 try writeByte(bw, byte);
374 return 1;
375}
376
377/// Returns 0 or 1 indicating how many bytes were written.
378/// `0` means end of stream encountered.
379pub fn writeByteUntilEnd(bw: *BufferedWriter, byte: u8) anyerror!usize {
289 const list = &bw.buffer;380 const list = &bw.buffer;
290 const buffer = list.items;381 const buffer = list.items;
291 if (buffer.len < list.capacity) {382 if (buffer.len < list.capacity) {
292 @branchHint(.likely);383 @branchHint(.likely);
293 buffer.ptr[buffer.len] = byte;384 buffer.ptr[buffer.len] = byte;
294 list.items.len = buffer.len + 1;385 list.items.len = buffer.len + 1;
295 return;386 return 1;
296 }387 }
297 var buffers: [2][]const u8 = .{ buffer, &.{byte} };388 var buffers: [2][]const u8 = .{ buffer, &.{byte} };
298 while (true) {389 while (true) {
299 const n = try bw.unbuffered_writer.writev(&buffers);390 const result = bw.unbuffered_writer.writev(&buffers);
391 try result.err;
392 const n = result.len;
300 if (n == 0) {393 if (n == 0) {
301 @branchHint(.unlikely);394 @branchHint(.unlikely);
395 if (result.end) return 0;
302 continue;396 continue;
303 } else if (n >= buffer.len) {397 } else if (n >= buffer.len) {
304 @branchHint(.likely);398 @branchHint(.likely);
305 if (n > buffer.len) {399 if (n > buffer.len) {
306 @branchHint(.likely);400 @branchHint(.likely);
307 list.items.len = 0;401 list.items.len = 0;
308 return;402 return 1;
309 } else {403 } else {
310 buffer[0] = byte;404 buffer[0] = byte;
311 list.items.len = 1;405 list.items.len = 1;
312 return;406 return 1;
313 }407 }
314 }408 }
315 const remainder = buffer[n..];409 const remainder = buffer[n..];
316 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);410 std.mem.copyForwards(u8, buffer[0..remainder.len], remainder);
317 buffer[remainder.len] = byte;411 buffer[remainder.len] = byte;
318 list.items.len = remainder.len + 1;412 list.items.len = remainder.len + 1;
319 return;413 return 1;
320 }414 }
321}415}
322416
323/// Writes the same byte many times, performing the underlying write call as417/// Writes the same byte many times, performing the underlying write call as
324/// many times as necessary.418/// many times as necessary, returning `error.WriteStreamEnd` if the byte
419/// could not be repeated `n` times.
325pub fn splatByteAll(bw: *BufferedWriter, byte: u8, n: usize) anyerror!void {420pub fn splatByteAll(bw: *BufferedWriter, byte: u8, n: usize) anyerror!void {
326 var remaining: usize = n;421 if ((try splatByteUntilEnd(bw, byte, n)) != n) return error.WriteStreamEnd;
327 while (remaining > 0) remaining -= try splatByte(bw, byte, remaining);422}
423
424/// Writes the same byte many times, performing the underlying write call as
425/// many times as necessary, returning `error.WriteStreamEnd` if the byte
426/// could not be repeated `n` times, or returning `n` on success.
427pub fn splatByteAllCount(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {
428 try splatByteAll(bw, byte, n);
429 return n;
430}
431
432/// Writes the same byte many times, performing the underlying write call as
433/// many times as necessary.
434///
435/// If the number returned is less than `n` it indicates end of stream.
436pub fn splatByteUntilEnd(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {
437 var index: usize = 0;
438 while (true) {
439 const result = splatByte(bw, byte, n - index);
440 try result.err;
441 index += result.len;
442 assert(index <= n);
443 if (index == n or result.end) return index;
444 }
328}445}
329446
330/// Writes the same byte many times, allowing short writes.447/// Writes the same byte many times, allowing short writes.
331///448///
332/// Does maximum of one underlying `Writer.VTable.writev`.449/// Does maximum of one underlying `Writer.VTable.writeSplat`.
333pub fn splatByte(bw: *BufferedWriter, byte: u8, n: usize) anyerror!usize {450pub fn splatByte(bw: *BufferedWriter, byte: u8, n: usize) Writer.Result {
334 return passthru_writeSplat(bw, &.{&.{byte}}, n);451 return passthru_writeSplat(bw, &.{&.{byte}}, n);
335}452}
336453
...@@ -389,7 +506,7 @@ pub fn writeFile(...@@ -389,7 +506,7 @@ pub fn writeFile(
389}506}
390507
391fn passthru_writeFile(508fn passthru_writeFile(
392 context: *anyopaque,509 context: ?*anyopaque,
393 file: std.fs.File,510 file: std.fs.File,
394 offset: u64,511 offset: u64,
395 len: Writer.FileLen,512 len: Writer.FileLen,
...@@ -544,32 +661,34 @@ pub fn alignBuffer(...@@ -544,32 +661,34 @@ pub fn alignBuffer(
544 width: usize,661 width: usize,
545 alignment: std.fmt.Alignment,662 alignment: std.fmt.Alignment,
546 fill: u8,663 fill: u8,
547) anyerror!void {664) anyerror!usize {
548 const padding = if (buffer.len < width) width - buffer.len else 0;665 const padding = if (buffer.len < width) width - buffer.len else 0;
549 if (padding == 0) {666 if (padding == 0) {
550 @branchHint(.likely);667 @branchHint(.likely);
551 return bw.writeAll(buffer);668 return bw.writeAllCount(buffer);
552 }669 }
670 var n: usize = 0;
553 switch (alignment) {671 switch (alignment) {
554 .left => {672 .left => {
555 try bw.writeAll(buffer);673 n += try bw.writeAllCount(buffer);
556 try bw.splatByteAll(fill, padding);674 n += try bw.splatByteAllCount(fill, padding);
557 },675 },
558 .center => {676 .center => {
559 const left_padding = padding / 2;677 const left_padding = padding / 2;
560 const right_padding = (padding + 1) / 2;678 const right_padding = (padding + 1) / 2;
561 try bw.splatByteAll(fill, left_padding);679 n += try bw.splatByteAllCount(fill, left_padding);
562 try bw.writeAll(buffer);680 n += try bw.writeAllCount(buffer);
563 try bw.splatByteAll(fill, right_padding);681 n += try bw.splatByteAllCount(fill, right_padding);
564 },682 },
565 .right => {683 .right => {
566 try bw.splatByteAll(fill, padding);684 n += try bw.splatByteAllCount(fill, padding);
567 try bw.writeAll(buffer);685 n += try bw.writeAllCount(buffer);
568 },686 },
569 }687 }
688 return n;
570}689}
571690
572pub fn alignBufferOptions(bw: *BufferedWriter, buffer: []const u8, options: std.fmt.Options) anyerror!void {691pub fn alignBufferOptions(bw: *BufferedWriter, buffer: []const u8, options: std.fmt.Options) anyerror!usize {
573 return alignBuffer(bw, buffer, options.width orelse buffer.len, options.alignment, options.fill);692 return alignBuffer(bw, buffer, options.width orelse buffer.len, options.alignment, options.fill);
574}693}
575694
...@@ -604,7 +723,7 @@ pub fn printValue(...@@ -604,7 +723,7 @@ pub fn printValue(
604 options: std.fmt.Options,723 options: std.fmt.Options,
605 value: anytype,724 value: anytype,
606 max_depth: usize,725 max_depth: usize,
607) anyerror!void {726) anyerror!usize {
608 const T = @TypeOf(value);727 const T = @TypeOf(value);
609 const actual_fmt = comptime if (std.mem.eql(u8, fmt, ANY))728 const actual_fmt = comptime if (std.mem.eql(u8, fmt, ANY))
610 defaultFormatString(T)729 defaultFormatString(T)
...@@ -619,13 +738,10 @@ pub fn printValue(...@@ -619,13 +738,10 @@ pub fn printValue(
619738
620 if (std.meta.hasMethod(T, "format")) {739 if (std.meta.hasMethod(T, "format")) {
621 if (fmt.len > 0 and fmt[0] == 'f') {740 if (fmt.len > 0 and fmt[0] == 'f') {
622 return value.format(fmt[1..], options, bw);741 return value.format(bw, fmt[1..]);
623 } else {742 } else if (fmt.len == 0) {
624 //@deprecated();743 // after 0.15.0 is tagged, delete the hasMethod condition and this compile error
625 // After 0.14.0 is tagged, uncomment this next line:744 @compileError("ambiguous format string; specify {f} to call format method, or {any} to skip it");
626 //@compileError("ambiguous format string; specify {f} to call format method, or {any} to skip it");
627 //and then delete the `hasMethod` condition
628 return value.format(fmt, options, bw);
629 }745 }
630 }746 }
631747
...@@ -662,92 +778,104 @@ pub fn printValue(...@@ -662,92 +778,104 @@ pub fn printValue(
662 },778 },
663 .error_set => {779 .error_set => {
664 if (actual_fmt.len > 0 and actual_fmt[0] == 's') {780 if (actual_fmt.len > 0 and actual_fmt[0] == 's') {
665 return bw.writeAll(@errorName(value));781 return bw.writeAllCount(@errorName(value));
666 } else if (actual_fmt.len != 0) {782 } else if (actual_fmt.len != 0) {
667 invalidFmtError(fmt, value);783 invalidFmtError(fmt, value);
668 } else {784 } else {
669 try bw.writeAll("error.");785 var n: usize = 0;
670 return bw.writeAll(@errorName(value));786 n += try bw.writeAllCount("error.");
787 n += try bw.writeAllCount(@errorName(value));
788 return n;
671 }789 }
672 },790 },
673 .@"enum" => |enumInfo| {791 .@"enum" => |enum_info| {
674 try bw.writeAll(@typeName(T));792 var n: usize = 0;
675 if (enumInfo.is_exhaustive) {793 n += try bw.writeAllCount(@typeName(T));
794 if (enum_info.is_exhaustive) {
676 if (actual_fmt.len != 0) invalidFmtError(fmt, value);795 if (actual_fmt.len != 0) invalidFmtError(fmt, value);
677 try bw.writeAll(".");796 n += try bw.writeAllCount(".");
678 try bw.writeAll(@tagName(value));797 n += try bw.writeAllCount(@tagName(value));
679 return;798 return n;
680 }799 }
681800
682 // Use @tagName only if value is one of known fields801 // Use @tagName only if value is one of known fields
683 @setEvalBranchQuota(3 * enumInfo.fields.len);802 @setEvalBranchQuota(3 * enum_info.fields.len);
684 inline for (enumInfo.fields) |enumField| {803 inline for (enum_info.fields) |enumField| {
685 if (@intFromEnum(value) == enumField.value) {804 if (@intFromEnum(value) == enumField.value) {
686 try bw.writeAll(".");805 n += try bw.writeAllCount(".");
687 try bw.writeAll(@tagName(value));806 n += try bw.writeAllCount(@tagName(value));
688 return;807 return;
689 }808 }
690 }809 }
691810
692 try bw.writeByte('(');811 n += try bw.writeByteCount('(');
693 try printValue(bw, actual_fmt, options, @intFromEnum(value), max_depth);812 n += try printValue(bw, actual_fmt, options, @intFromEnum(value), max_depth);
694 try bw.writeByte(')');813 n += try bw.writeByteCount(')');
814 return n;
695 },815 },
696 .@"union" => |info| {816 .@"union" => |info| {
697 if (actual_fmt.len != 0) invalidFmtError(fmt, value);817 if (actual_fmt.len != 0) invalidFmtError(fmt, value);
698 try bw.writeAll(@typeName(T));818 var n: usize = 0;
819 n += try bw.writeAllCount(@typeName(T));
699 if (max_depth == 0) {820 if (max_depth == 0) {
700 return bw.writeAll("{ ... }");821 n += bw.writeAllCount("{ ... }");
822 return n;
701 }823 }
702 if (info.tag_type) |UnionTagType| {824 if (info.tag_type) |UnionTagType| {
703 try bw.writeAll("{ .");825 n += try bw.writeAllCount("{ .");
704 try bw.writeAll(@tagName(@as(UnionTagType, value)));826 n += try bw.writeAllCount(@tagName(@as(UnionTagType, value)));
705 try bw.writeAll(" = ");827 n += try bw.writeAllCount(" = ");
706 inline for (info.fields) |u_field| {828 inline for (info.fields) |u_field| {
707 if (value == @field(UnionTagType, u_field.name)) {829 if (value == @field(UnionTagType, u_field.name)) {
708 try printValue(bw, ANY, options, @field(value, u_field.name), max_depth - 1);830 n += try printValue(bw, ANY, options, @field(value, u_field.name), max_depth - 1);
709 }831 }
710 }832 }
711 try bw.writeAll(" }");833 n += try bw.writeAllCount(" }");
712 } else {834 } else {
713 try bw.writeByte('@');835 n += try bw.writeByte('@');
714 try bw.printIntOptions(@intFromPtr(&value), 16, .lower);836 n += try bw.printIntOptions(@intFromPtr(&value), 16, .lower);
715 }837 }
838 return n;
716 },839 },
717 .@"struct" => |info| {840 .@"struct" => |info| {
718 if (actual_fmt.len != 0) invalidFmtError(fmt, value);841 if (actual_fmt.len != 0) invalidFmtError(fmt, value);
842 var n: usize = 0;
719 if (info.is_tuple) {843 if (info.is_tuple) {
720 // Skip the type and field names when formatting tuples.844 // Skip the type and field names when formatting tuples.
721 if (max_depth == 0) {845 if (max_depth == 0) {
722 return bw.writeAll("{ ... }");846 n += try bw.writeAllCount("{ ... }");
847 return n;
723 }848 }
724 try bw.writeAll("{");849 n += try bw.writeAllCount("{");
725 inline for (info.fields, 0..) |f, i| {850 inline for (info.fields, 0..) |f, i| {
726 if (i == 0) {851 if (i == 0) {
727 try bw.writeAll(" ");852 n += try bw.writeAllCount(" ");
728 } else {853 } else {
729 try bw.writeAll(", ");854 n += try bw.writeAllCount(", ");
730 }855 }
731 try printValue(bw, ANY, options, @field(value, f.name), max_depth - 1);856 n += try printValue(bw, ANY, options, @field(value, f.name), max_depth - 1);
732 }857 }
733 return bw.writeAll(" }");858 n += try bw.writeAllCount(" }");
859 return n;
734 }860 }
735 try bw.writeAll(@typeName(T));861 n += try bw.writeAllCount(@typeName(T));
736 if (max_depth == 0) {862 if (max_depth == 0) {
737 return bw.writeAll("{ ... }");863 n += try bw.writeAllCount("{ ... }");
864 return n;
738 }865 }
739 try bw.writeAll("{");866 n += try bw.writeAllCount("{");
740 inline for (info.fields, 0..) |f, i| {867 inline for (info.fields, 0..) |f, i| {
741 if (i == 0) {868 if (i == 0) {
742 try bw.writeAll(" .");869 n += try bw.writeAllCount(" .");
743 } else {870 } else {
744 try bw.writeAll(", .");871 n += try bw.writeAllCount(", .");
745 }872 }
746 try bw.writeAll(f.name);873 n += try bw.writeAllCount(f.name);
747 try bw.writeAll(" = ");874 n += try bw.writeAllCount(" = ");
748 try printValue(bw, ANY, options, @field(value, f.name), max_depth - 1);875 n += try printValue(bw, ANY, options, @field(value, f.name), max_depth - 1);
749 }876 }
750 try bw.writeAll(" }");877 n += try bw.writeAllCount(" }");
878 return n;
751 },879 },
752 .pointer => |ptr_info| switch (ptr_info.size) {880 .pointer => |ptr_info| switch (ptr_info.size) {
753 .one => switch (@typeInfo(ptr_info.child)) {881 .one => switch (@typeInfo(ptr_info.child)) {
...@@ -756,8 +884,10 @@ pub fn printValue(...@@ -756,8 +884,10 @@ pub fn printValue(
756 },884 },
757 else => {885 else => {
758 var buffers: [2][]const u8 = .{ @typeName(ptr_info.child), "@" };886 var buffers: [2][]const u8 = .{ @typeName(ptr_info.child), "@" };
759 try writevAll(bw, &buffers);887 var n: usize = 0;
760 try printIntOptions(bw, @intFromPtr(value), 16, .lower, options);888 n += try writevAll(bw, &buffers);
889 n += try printIntOptions(bw, @intFromPtr(value), 16, .lower, options);
890 return n;
761 },891 },
762 },892 },
763 .many, .c => {893 .many, .c => {
...@@ -775,7 +905,7 @@ pub fn printValue(...@@ -775,7 +905,7 @@ pub fn printValue(
775 if (actual_fmt.len == 0)905 if (actual_fmt.len == 0)
776 @compileError("cannot format slice without a specifier (i.e. {s}, {x}, {b64}, or {any})");906 @compileError("cannot format slice without a specifier (i.e. {s}, {x}, {b64}, or {any})");
777 if (max_depth == 0) {907 if (max_depth == 0) {
778 return bw.writeAll("{ ... }");908 return bw.writeAllCount("{ ... }");
779 }909 }
780 if (ptr_info.child == u8) switch (actual_fmt.len) {910 if (ptr_info.child == u8) switch (actual_fmt.len) {
781 1 => switch (actual_fmt[0]) {911 1 => switch (actual_fmt[0]) {
...@@ -789,21 +919,23 @@ pub fn printValue(...@@ -789,21 +919,23 @@ pub fn printValue(
789 },919 },
790 else => {},920 else => {},
791 };921 };
792 try bw.writeAll("{ ");922 var n: usize = 0;
923 n += try bw.writeAllCount("{ ");
793 for (value, 0..) |elem, i| {924 for (value, 0..) |elem, i| {
794 try printValue(bw, actual_fmt, options, elem, max_depth - 1);925 n += try printValue(bw, actual_fmt, options, elem, max_depth - 1);
795 if (i != value.len - 1) {926 if (i != value.len - 1) {
796 try bw.writeAll(", ");927 n += try bw.writeAllCount(", ");
797 }928 }
798 }929 }
799 try bw.writeAll(" }");930 n += try bw.writeAllCount(" }");
931 return n;
800 },932 },
801 },933 },
802 .array => |info| {934 .array => |info| {
803 if (actual_fmt.len == 0)935 if (actual_fmt.len == 0)
804 @compileError("cannot format array without a specifier (i.e. {s} or {any})");936 @compileError("cannot format array without a specifier (i.e. {s} or {any})");
805 if (max_depth == 0) {937 if (max_depth == 0) {
806 return bw.writeAll("{ ... }");938 return bw.writeAllCount("{ ... }");
807 }939 }
808 if (info.child == u8) {940 if (info.child == u8) {
809 if (actual_fmt[0] == 's') {941 if (actual_fmt[0] == 's') {
...@@ -814,28 +946,32 @@ pub fn printValue(...@@ -814,28 +946,32 @@ pub fn printValue(
814 return printHex(bw, &value, .upper);946 return printHex(bw, &value, .upper);
815 }947 }
816 }948 }
817 try bw.writeAll("{ ");949 var n: usize = 0;
950 n += try bw.writeAllCount("{ ");
818 for (value, 0..) |elem, i| {951 for (value, 0..) |elem, i| {
819 try printValue(bw, actual_fmt, options, elem, max_depth - 1);952 n += try printValue(bw, actual_fmt, options, elem, max_depth - 1);
820 if (i < value.len - 1) {953 if (i < value.len - 1) {
821 try bw.writeAll(", ");954 n += try bw.writeAllCount(", ");
822 }955 }
823 }956 }
824 try bw.writeAll(" }");957 n += try bw.writeAllCount(" }");
958 return n;
825 },959 },
826 .vector => |info| {960 .vector => |info| {
827 if (max_depth == 0) {961 if (max_depth == 0) {
828 return bw.writeAll("{ ... }");962 return bw.writeAllCount("{ ... }");
829 }963 }
830 try bw.writeAll("{ ");964 var n: usize = 0;
965 n += try bw.writeAllCount("{ ");
831 var i: usize = 0;966 var i: usize = 0;
832 while (i < info.len) : (i += 1) {967 while (i < info.len) : (i += 1) {
833 try printValue(bw, actual_fmt, options, value[i], max_depth - 1);968 n += try printValue(bw, actual_fmt, options, value[i], max_depth - 1);
834 if (i < info.len - 1) {969 if (i < info.len - 1) {
835 try bw.writeAll(", ");970 n += try bw.writeAllCount(", ");
836 }971 }
837 }972 }
838 try bw.writeAll(" }");973 n += try bw.writeAllCount(" }");
974 return n;
839 },975 },
840 .@"fn" => @compileError("unable to format function body type, use '*const " ++ @typeName(T) ++ "' for a function pointer type"),976 .@"fn" => @compileError("unable to format function body type, use '*const " ++ @typeName(T) ++ "' for a function pointer type"),
841 .type => {977 .type => {
...@@ -860,7 +996,7 @@ pub fn printInt(...@@ -860,7 +996,7 @@ pub fn printInt(
860 comptime fmt: []const u8,996 comptime fmt: []const u8,
861 options: std.fmt.Options,997 options: std.fmt.Options,
862 value: anytype,998 value: anytype,
863) anyerror!void {999) anyerror!usize {
864 const int_value = if (@TypeOf(value) == comptime_int) blk: {1000 const int_value = if (@TypeOf(value) == comptime_int) blk: {
865 const Int = std.math.IntFittingRange(value, value);1001 const Int = std.math.IntFittingRange(value, value);
866 break :blk @as(Int, value);1002 break :blk @as(Int, value);
...@@ -904,15 +1040,15 @@ pub fn printInt(...@@ -904,15 +1040,15 @@ pub fn printInt(
904 comptime unreachable;1040 comptime unreachable;
905}1041}
9061042
907pub fn printAsciiChar(bw: *BufferedWriter, c: u8, options: std.fmt.Options) anyerror!void {1043pub fn printAsciiChar(bw: *BufferedWriter, c: u8, options: std.fmt.Options) anyerror!usize {
908 return alignBufferOptions(bw, @as(*const [1]u8, &c), options);1044 return alignBufferOptions(bw, @as(*const [1]u8, &c), options);
909}1045}
9101046
911pub fn printAscii(bw: *BufferedWriter, bytes: []const u8, options: std.fmt.Options) anyerror!void {1047pub fn printAscii(bw: *BufferedWriter, bytes: []const u8, options: std.fmt.Options) anyerror!usize {
912 return alignBufferOptions(bw, bytes, options);1048 return alignBufferOptions(bw, bytes, options);
913}1049}
9141050
915pub fn printUnicodeCodepoint(bw: *BufferedWriter, c: u21, options: std.fmt.Options) anyerror!void {1051pub fn printUnicodeCodepoint(bw: *BufferedWriter, c: u21, options: std.fmt.Options) anyerror!usize {
916 var buf: [4]u8 = undefined;1052 var buf: [4]u8 = undefined;
917 const len = try std.unicode.utf8Encode(c, &buf);1053 const len = try std.unicode.utf8Encode(c, &buf);
918 return alignBufferOptions(bw, buf[0..len], options);1054 return alignBufferOptions(bw, buf[0..len], options);
...@@ -924,7 +1060,7 @@ pub fn printIntOptions(...@@ -924,7 +1060,7 @@ pub fn printIntOptions(
924 base: u8,1060 base: u8,
925 case: std.fmt.Case,1061 case: std.fmt.Case,
926 options: std.fmt.Options,1062 options: std.fmt.Options,
927) anyerror!void {1063) anyerror!usize {
928 assert(base >= 2);1064 assert(base >= 2);
9291065
930 const int_value = if (@TypeOf(value) == comptime_int) blk: {1066 const int_value = if (@TypeOf(value) == comptime_int) blk: {
...@@ -991,7 +1127,7 @@ pub fn printFloat(...@@ -991,7 +1127,7 @@ pub fn printFloat(
991 comptime fmt: []const u8,1127 comptime fmt: []const u8,
992 options: std.fmt.Options,1128 options: std.fmt.Options,
993 value: anytype,1129 value: anytype,
994) anyerror!void {1130) anyerror!usize {
995 var buf: [std.fmt.float.bufferSize(.decimal, f64)]u8 = undefined;1131 var buf: [std.fmt.float.bufferSize(.decimal, f64)]u8 = undefined;
9961132
997 if (fmt.len > 1) invalidFmtError(fmt, value);1133 if (fmt.len > 1) invalidFmtError(fmt, value);
...@@ -1279,7 +1415,7 @@ pub fn printDuration(bw: *BufferedWriter, nanoseconds: anytype, options: std.fmt...@@ -1279,7 +1415,7 @@ pub fn printDuration(bw: *BufferedWriter, nanoseconds: anytype, options: std.fmt
1279 return alignBufferOptions(bw, sub_bw.getWritten(), options);1415 return alignBufferOptions(bw, sub_bw.getWritten(), options);
1280}1416}
12811417
1282pub fn printHex(bw: *BufferedWriter, bytes: []const u8, case: std.fmt.Case) anyerror!void {1418pub fn printHex(bw: *BufferedWriter, bytes: []const u8, case: std.fmt.Case) anyerror!usize {
1283 const charset = switch (case) {1419 const charset = switch (case) {
1284 .upper => "0123456789ABCDEF",1420 .upper => "0123456789ABCDEF",
1285 .lower => "0123456789abcdef",1421 .lower => "0123456789abcdef",
...@@ -1288,12 +1424,68 @@ pub fn printHex(bw: *BufferedWriter, bytes: []const u8, case: std.fmt.Case) anye...@@ -1288,12 +1424,68 @@ pub fn printHex(bw: *BufferedWriter, bytes: []const u8, case: std.fmt.Case) anye
1288 try writeByte(bw, charset[c >> 4]);1424 try writeByte(bw, charset[c >> 4]);
1289 try writeByte(bw, charset[c & 15]);1425 try writeByte(bw, charset[c & 15]);
1290 }1426 }
1427 return bytes.len * 2;
1291}1428}
12921429
1293pub fn printBase64(bw: *BufferedWriter, bytes: []const u8) anyerror!void {1430pub fn printBase64(bw: *BufferedWriter, bytes: []const u8) anyerror!usize {
1294 var chunker = std.mem.window(u8, bytes, 3, 3);1431 var chunker = std.mem.window(u8, bytes, 3, 3);
1295 var temp: [5]u8 = undefined;1432 var temp: [5]u8 = undefined;
1296 while (chunker.next()) |chunk| try bw.writeAll(std.base64.standard.Encoder.encode(&temp, chunk));1433 var n: usize = 0;
1434 while (chunker.next()) |chunk| {
1435 n += try bw.writeAllCount(std.base64.standard.Encoder.encode(&temp, chunk));
1436 }
1437 return n;
1438}
1439
1440/// Write a single unsigned integer as unsigned LEB128 to the given writer.
1441pub fn writeUleb128(bw: *std.io.BufferedWriter, arg: anytype) anyerror!usize {
1442 const Arg = @TypeOf(arg);
1443 const Int = switch (Arg) {
1444 comptime_int => std.math.IntFittingRange(arg, arg),
1445 else => Arg,
1446 };
1447 const Value = if (@typeInfo(Int).int.bits < 8) u8 else Int;
1448 var value: Value = arg;
1449 var n: usize = 0;
1450
1451 while (true) {
1452 const byte: u8 = @truncate(value & 0x7f);
1453 value >>= 7;
1454 if (value == 0) {
1455 try bw.writeByte(byte);
1456 return n + 1;
1457 } else {
1458 try bw.writeByte(byte | 0x80);
1459 n += 1;
1460 }
1461 }
1462}
1463
1464/// Write a single signed integer as signed LEB128 to the given writer.
1465pub fn writeIleb128(bw: *std.io.BufferedWriter, arg: anytype) anyerror!usize {
1466 const Arg = @TypeOf(arg);
1467 const Int = switch (Arg) {
1468 comptime_int => std.math.IntFittingRange(-@abs(arg), @abs(arg)),
1469 else => Arg,
1470 };
1471 const Signed = if (@typeInfo(Int).int.bits < 8) i8 else Int;
1472 const Unsigned = std.meta.Int(.unsigned, @typeInfo(Signed).int.bits);
1473 var value: Signed = arg;
1474 var n: usize = 0;
1475
1476 while (true) {
1477 const unsigned: Unsigned = @bitCast(value);
1478 const byte: u8 = @truncate(unsigned);
1479 value >>= 6;
1480 if (value == -1 or value == 0) {
1481 try bw.writeByte(byte & 0x7F);
1482 return n + 1;
1483 } else {
1484 value >>= 1;
1485 try bw.writeByte(byte | 0x80);
1486 n += 1;
1487 }
1488 }
1297}1489}
12981490
1299test "formatValue max_depth" {1491test "formatValue max_depth" {
...@@ -1590,15 +1782,15 @@ test "fixed output" {...@@ -1590,15 +1782,15 @@ test "fixed output" {
1590 try bw.writeAll("world");1782 try bw.writeAll("world");
1591 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Helloworld"));1783 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Helloworld"));
15921784
1593 try testing.expectError(error.NoSpaceLeft, bw.writeAll("!"));1785 try testing.expectError(error.WriteStreamEnd, bw.writeAll("!"));
1594 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Helloworld"));1786 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Helloworld"));
15951787
1596 bw.reset();1788 bw.reset();
1597 try testing.expect(bw.getWritten().len == 0);1789 try testing.expect(bw.getWritten().len == 0);
15981790
1599 try testing.expectError(error.NoSpaceLeft, bw.writeAll("Hello world!"));1791 try testing.expectError(error.WriteStreamEnd, bw.writeAll("Hello world!"));
1600 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Hello worl"));1792 try testing.expect(std.mem.eql(u8, bw.getWritten(), "Hello worl"));
16011793
1602 try bw.seekTo((try bw.getEndPos()) + 1);1794 try bw.seekTo((try bw.getEndPos()) + 1);
1603 try testing.expectError(error.NoSpaceLeft, bw.writeAll("H"));1795 try testing.expectError(error.WriteStreamEnd, bw.writeAll("H"));
1604}1796}
lib/std/io/CountingReader.zig deleted-29
...@@ -1,29 +0,0 @@
1//! A Reader that counts how many bytes has been read from it.
2
3const std = @import("../std.zig");
4const CountingReader = @This();
5
6child_reader: std.io.Reader,
7bytes_read: u64 = 0,
8
9pub fn read(self: *@This(), buf: []u8) anyerror!usize {
10 const amt = try self.child_reader.read(buf);
11 self.bytes_read += amt;
12 return amt;
13}
14
15pub fn reader(self: *@This()) std.io.Reader {
16 return .{ .context = self };
17}
18
19test CountingReader {
20 const bytes = "yay" ** 20;
21 var fbs: std.io.BufferedReader = undefined;
22 fbs.initFixed(bytes);
23 var counting_stream: CountingReader = .{ .child_reader = fbs.reader() };
24 var stream = counting_stream.reader().unbuffered();
25 while (stream.readByte()) |_| {} else |err| {
26 try std.testing.expectError(error.EndOfStream, err);
27 }
28 try std.testing.expect(counting_stream.bytes_read == bytes.len);
29}
lib/std/io/CountingWriter.zig deleted-52
...@@ -1,52 +0,0 @@
1//! TODO make this more like AllocatingWriter, managing the state of
2//! BufferedWriter both as the output and the input, but with only
3//! one buffer.
4const std = @import("../std.zig");
5const CountingWriter = @This();
6const assert = std.debug.assert;
7const native_endian = @import("builtin").target.cpu.arch.endian();
8const Writer = std.io.Writer;
9const testing = std.testing;
10
11/// Underlying stream to passthrough bytes to.
12child_writer: Writer,
13bytes_written: u64 = 0,
14
15pub fn writer(cw: *CountingWriter) Writer {
16 return .{
17 .context = cw,
18 .vtable = &.{
19 .writeSplat = passthru_writeSplat,
20 .writeFile = passthru_writeFile,
21 },
22 };
23}
24
25fn passthru_writeSplat(context: *anyopaque, data: []const []const u8, splat: usize) anyerror!usize {
26 const cw: *CountingWriter = @alignCast(@ptrCast(context));
27 const n = try cw.child_writer.writeSplat(data, splat);
28 cw.bytes_written += n;
29 return n;
30}
31
32fn passthru_writeFile(
33 context: *anyopaque,
34 file: std.fs.File,
35 offset: u64,
36 len: Writer.FileLen,
37 headers_and_trailers: []const []const u8,
38 headers_len: usize,
39) anyerror!usize {
40 const cw: *CountingWriter = @alignCast(@ptrCast(context));
41 const n = try cw.child_writer.writeFile(file, offset, len, headers_and_trailers, headers_len);
42 cw.bytes_written += n;
43 return n;
44}
45
46test CountingWriter {
47 var cw: CountingWriter = .{ .child_writer = std.io.null_writer };
48 var bw = cw.writer().unbuffered();
49 const bytes = "yay";
50 try bw.writeAll(bytes);
51 try testing.expect(cw.bytes_written == bytes.len);
52}
lib/std/io/Reader.zig+51-41
...@@ -19,8 +19,8 @@ pub const VTable = struct {...@@ -19,8 +19,8 @@ pub const VTable = struct {
19 ///19 ///
20 /// If this is `null` it is equivalent to always returning20 /// If this is `null` it is equivalent to always returning
21 /// `error.Unseekable`.21 /// `error.Unseekable`.
22 posRead: ?*const fn (ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit, offset: u64) Result,22 posRead: ?*const fn (ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit, offset: u64) RwResult,
23 posReadVec: ?*const fn (ctx: ?*anyopaque, data: []const []u8, offset: u64) VecResult,23 posReadVec: ?*const fn (ctx: ?*anyopaque, data: []const []u8, offset: u64) Result,
2424
25 /// Writes bytes from the internally tracked stream position to `bw`, or25 /// Writes bytes from the internally tracked stream position to `bw`, or
26 /// returns `error.Unstreamable`, indicating `posRead` should be used26 /// returns `error.Unstreamable`, indicating `posRead` should be used
...@@ -37,38 +37,34 @@ pub const VTable = struct {...@@ -37,38 +37,34 @@ pub const VTable = struct {
37 ///37 ///
38 /// If this is `null` it is equivalent to always returning38 /// If this is `null` it is equivalent to always returning
39 /// `error.Unstreamable`.39 /// `error.Unstreamable`.
40 streamRead: ?*const fn (ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit) Result,40 streamRead: ?*const fn (ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit) RwResult,
41 streamReadVec: ?*const fn (ctx: ?*anyopaque, data: []const []u8) VecResult,41 streamReadVec: ?*const fn (ctx: ?*anyopaque, data: []const []u8) Result,
42};42
4343 pub const eof: VTable = .{
44pub const Len = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(usize) - 1 } });44 .posRead = eof_posRead,
4545 .posReadVec = eof_posReadVec,
46pub const VecResult = struct {46 .streamRead = eof_streamRead,
47 /// Even when a failure occurs, `Effect.written` may be nonzero, and47 .streamReadVec = eof_streamReadVec,
48 /// `Effect.end` may be true.48 };
49 failure: anyerror!void,
50 effect: VecEffect,
51};49};
5250
53pub const Result = struct {51pub const Result = std.io.Writer.Result;
54 /// Even when a failure occurs, `Effect.written` may be nonzero, and
55 /// `Effect.end` may be true.
56 failure: anyerror!void,
57 write_effect: Effect,
58 read_effect: Effect,
59};
6052
61pub const Effect = packed struct(usize) {53pub const RwResult = struct {
62 /// Number of bytes that were read from the reader or written to the54 len: usize = 0,
63 /// writer.55 read_err: anyerror!void = {},
64 len: Len,56 write_err: anyerror!void = {},
65 /// Indicates end of stream.57 read_end: bool = false,
66 end: bool,58 write_end: bool = false,
67};59};
6860
69pub const Limit = enum(usize) {61pub const Limit = enum(usize) {
70 none = std.math.maxInt(usize),62 none = std.math.maxInt(usize),
71 _,63 _,
64
65 pub fn min(l: Limit, int: usize) usize {
66 return @min(int, @intFromEnum(l));
67 }
72};68};
7369
74/// Returns total number of bytes written to `w`.70/// Returns total number of bytes written to `w`.
...@@ -133,25 +129,11 @@ pub fn streamReadAlloc(r: Reader, gpa: std.mem.Allocator, max_size: usize) anyer...@@ -133,25 +129,11 @@ pub fn streamReadAlloc(r: Reader, gpa: std.mem.Allocator, max_size: usize) anyer
133129
134/// Reads the stream until the end, ignoring all the data.130/// Reads the stream until the end, ignoring all the data.
135/// Returns the number of bytes discarded.131/// Returns the number of bytes discarded.
136pub fn discardAll(r: Reader) anyerror!usize {132pub fn discardUntilEnd(r: Reader) anyerror!usize {
137 var bw = std.io.null_writer.unbuffered();133 var bw = std.io.null_writer.unbuffered();
138 return streamReadAll(r, &bw);134 return streamReadAll(r, &bw);
139}135}
140136
141pub fn buffered(r: Reader, buffer: []u8) std.io.BufferedReader {
142 return .{
143 .reader = r,
144 .buffered_writer = .{
145 .buffer = buffer,
146 .mode = .fixed,
147 },
148 };
149}
150
151pub fn unbuffered(r: Reader) std.io.BufferedReader {
152 return buffered(r, &.{});
153}
154
155pub fn allocating(r: Reader, gpa: std.mem.Allocator) std.io.BufferedReader {137pub fn allocating(r: Reader, gpa: std.mem.Allocator) std.io.BufferedReader {
156 return .{138 return .{
157 .reader = r,139 .reader = r,
...@@ -189,3 +171,31 @@ test "when the backing reader provides one byte at a time" {...@@ -189,3 +171,31 @@ test "when the backing reader provides one byte at a time" {
189 defer std.testing.allocator.free(res);171 defer std.testing.allocator.free(res);
190 try std.testing.expectEqualStrings(str, res);172 try std.testing.expectEqualStrings(str, res);
191}173}
174
175fn eof_posRead(ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit, offset: u64) RwResult {
176 _ = ctx;
177 _ = bw;
178 _ = limit;
179 _ = offset;
180 return .{ .end = true };
181}
182
183fn eof_posReadVec(ctx: ?*anyopaque, data: []const []u8, offset: u64) Result {
184 _ = ctx;
185 _ = data;
186 _ = offset;
187 return .{ .end = true };
188}
189
190fn eof_streamRead(ctx: ?*anyopaque, bw: *std.io.BufferedWriter, limit: Limit) RwResult {
191 _ = ctx;
192 _ = bw;
193 _ = limit;
194 return .{ .end = true };
195}
196
197fn eof_streamReadVec(ctx: ?*anyopaque, data: []const []u8) Result {
198 _ = ctx;
199 _ = data;
200 return .{ .end = true };
201}
lib/std/io/Writer.zig+98-21
...@@ -2,7 +2,7 @@ const std = @import("../std.zig");...@@ -2,7 +2,7 @@ const std = @import("../std.zig");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const Writer = @This();3const Writer = @This();
44
5context: *anyopaque,5context: ?*anyopaque,
6vtable: *const VTable,6vtable: *const VTable,
77
8pub const VTable = struct {8pub const VTable = struct {
...@@ -17,7 +17,7 @@ pub const VTable = struct {...@@ -17,7 +17,7 @@ pub const VTable = struct {
17 /// Number of bytes returned may be zero, which does not mean17 /// Number of bytes returned may be zero, which does not mean
18 /// end-of-stream. A subsequent call may return nonzero, or may signal end18 /// end-of-stream. A subsequent call may return nonzero, or may signal end
19 /// of stream via an error.19 /// of stream via an error.
20 writeSplat: *const fn (ctx: *anyopaque, data: []const []const u8, splat: usize) Result,20 writeSplat: *const fn (ctx: ?*anyopaque, data: []const []const u8, splat: usize) Result,
2121
22 /// Writes contents from an open file. `headers` are written first, then `len`22 /// Writes contents from an open file. `headers` are written first, then `len`
23 /// bytes of `file` starting from `offset`, then `trailers`.23 /// bytes of `file` starting from `offset`, then `trailers`.
...@@ -29,7 +29,7 @@ pub const VTable = struct {...@@ -29,7 +29,7 @@ pub const VTable = struct {
29 /// end-of-stream. A subsequent call may return nonzero, or may signal end29 /// end-of-stream. A subsequent call may return nonzero, or may signal end
30 /// of stream via an error.30 /// of stream via an error.
31 writeFile: *const fn (31 writeFile: *const fn (
32 ctx: *anyopaque,32 ctx: ?*anyopaque,
33 file: std.fs.File,33 file: std.fs.File,
34 offset: Offset,34 offset: Offset,
35 /// When zero, it means copy until the end of the file is reached.35 /// When zero, it means copy until the end of the file is reached.
...@@ -39,25 +39,26 @@ pub const VTable = struct {...@@ -39,25 +39,26 @@ pub const VTable = struct {
39 headers_and_trailers: []const []const u8,39 headers_and_trailers: []const []const u8,
40 headers_len: usize,40 headers_len: usize,
41 ) Result,41 ) Result,
42};
43
44pub const Len = @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(usize) - 1 } });
4542
46pub const Result = struct {43 pub const eof: VTable = .{
47 /// Even when a failure occurs, `Effect.written` may be nonzero, and44 .writeSplat = eof_writeSplat,
48 /// `Effect.end` may be true.45 .writeFile = eof_writeFile,
49 failure: anyerror!void,46 };
50 effect: Effect,
51};47};
5248
53pub const Effect = packed struct(usize) {49pub const Result = struct {
54 /// Number of bytes that were written to `writer`.50 /// Even when a failure occurs, `len` may be nonzero, and `end` may be
55 len: Len,51 /// true.
52 err: anyerror!void = {},
53 /// Number of bytes that were transferred. When an error occurs, ideally
54 /// this will be zero, but may not always be the case.
55 len: usize = 0,
56 /// Indicates end of stream.56 /// Indicates end of stream.
57 end: bool,57 end: bool = false,
58};58};
5959
60pub const Offset = enum(u64) {60pub const Offset = enum(u64) {
61 /// Indicates to read the file as a stream.
61 none = std.math.maxInt(u64),62 none = std.math.maxInt(u64),
62 _,63 _,
6364
...@@ -66,6 +67,11 @@ pub const Offset = enum(u64) {...@@ -66,6 +67,11 @@ pub const Offset = enum(u64) {
66 assert(result != .none);67 assert(result != .none);
67 return result;68 return result;
68 }69 }
70
71 pub fn toInt(o: Offset) ?u64 {
72 if (o == .none) return null;
73 return @intFromEnum(o);
74 }
69};75};
7076
71pub const FileLen = enum(u64) {77pub const FileLen = enum(u64) {
...@@ -84,11 +90,11 @@ pub const FileLen = enum(u64) {...@@ -84,11 +90,11 @@ pub const FileLen = enum(u64) {
84 }90 }
85};91};
8692
87pub fn writev(w: Writer, data: []const []const u8) anyerror!usize {93pub fn writev(w: Writer, data: []const []const u8) Result {
88 return w.vtable.writeSplat(w.context, data, 1);94 return w.vtable.writeSplat(w.context, data, 1);
89}95}
9096
91pub fn writeSplat(w: Writer, data: []const []const u8, splat: usize) anyerror!usize {97pub fn writeSplat(w: Writer, data: []const []const u8, splat: usize) Result {
92 return w.vtable.writeSplat(w.context, data, splat);98 return w.vtable.writeSplat(w.context, data, splat);
93}99}
94100
...@@ -99,25 +105,25 @@ pub fn writeFile(...@@ -99,25 +105,25 @@ pub fn writeFile(
99 len: FileLen,105 len: FileLen,
100 headers_and_trailers: []const []const u8,106 headers_and_trailers: []const []const u8,
101 headers_len: usize,107 headers_len: usize,
102) anyerror!usize {108) Result {
103 return w.vtable.writeFile(w.context, file, offset, len, headers_and_trailers, headers_len);109 return w.vtable.writeFile(w.context, file, offset, len, headers_and_trailers, headers_len);
104}110}
105111
106pub fn unimplemented_writeFile(112pub fn unimplemented_writeFile(
107 context: *anyopaque,113 context: ?*anyopaque,
108 file: std.fs.File,114 file: std.fs.File,
109 offset: u64,115 offset: u64,
110 len: FileLen,116 len: FileLen,
111 headers_and_trailers: []const []const u8,117 headers_and_trailers: []const []const u8,
112 headers_len: usize,118 headers_len: usize,
113) anyerror!usize {119) Result {
114 _ = context;120 _ = context;
115 _ = file;121 _ = file;
116 _ = offset;122 _ = offset;
117 _ = len;123 _ = len;
118 _ = headers_and_trailers;124 _ = headers_and_trailers;
119 _ = headers_len;125 _ = headers_len;
120 return error.Unimplemented;126 return .{ .err = error.Unimplemented };
121}127}
122128
123pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {129pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {
...@@ -130,3 +136,74 @@ pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {...@@ -130,3 +136,74 @@ pub fn buffered(w: Writer, buffer: []u8) std.io.BufferedWriter {
130pub fn unbuffered(w: Writer) std.io.BufferedWriter {136pub fn unbuffered(w: Writer) std.io.BufferedWriter {
131 return buffered(w, &.{});137 return buffered(w, &.{});
132}138}
139
140/// A `Writer` that discards all data.
141pub const @"null": Writer = .{
142 .context = undefined,
143 .vtable = &.{
144 .writeSplat = null_writeSplat,
145 .writeFile = null_writeFile,
146 },
147};
148
149fn null_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Result {
150 _ = context;
151 const headers = data[0 .. data.len - 1];
152 const pattern = data[headers.len..];
153 var written: usize = pattern.len * splat;
154 for (headers) |bytes| written += bytes.len;
155 return .{ .len = written };
156}
157
158fn null_writeFile(
159 context: ?*anyopaque,
160 file: std.fs.File,
161 offset: Offset,
162 len: FileLen,
163 headers_and_trailers: []const []const u8,
164 headers_len: usize,
165) Result {
166 _ = context;
167 var n: usize = 0;
168 if (len == .entire_file) {
169 const headers = headers_and_trailers[0..headers_len];
170 for (headers) |bytes| n += bytes.len;
171 if (offset.toInt()) |off| {
172 const stat = file.stat() catch |err| return .{ .err = err, .len = n };
173 n += stat.size - off;
174 for (headers_and_trailers[headers_len..]) |bytes| n += bytes.len;
175 return .{ .len = n };
176 }
177 @panic("TODO stream from file until eof, counting");
178 }
179 for (headers_and_trailers) |bytes| n += bytes.len;
180 return .{ .len = len.int() + n };
181}
182
183test @"null" {
184 try @"null".writeAll("yay");
185}
186
187fn eof_writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Result {
188 _ = context;
189 _ = data;
190 _ = splat;
191 return .{ .end = true };
192}
193
194fn eof_writeFile(
195 context: ?*anyopaque,
196 file: std.fs.File,
197 offset: u64,
198 len: FileLen,
199 headers_and_trailers: []const []const u8,
200 headers_len: usize,
201) Result {
202 _ = context;
203 _ = file;
204 _ = offset;
205 _ = len;
206 _ = headers_and_trailers;
207 _ = headers_len;
208 return .{ .end = true };
209}
lib/std/leb128.zig+31-175
...@@ -2,151 +2,6 @@ const builtin = @import("builtin");...@@ -2,151 +2,6 @@ const builtin = @import("builtin");
2const std = @import("std");2const std = @import("std");
3const testing = std.testing;3const testing = std.testing;
44
5/// Read a single unsigned LEB128 value from the given reader as type T,
6/// or error.Overflow if the value cannot fit.
7pub fn readUleb128(comptime T: type, reader: anytype) !T {
8 const U = if (@typeInfo(T).int.bits < 8) u8 else T;
9 const ShiftT = std.math.Log2Int(U);
10
11 const max_group = (@typeInfo(U).int.bits + 6) / 7;
12
13 var value: U = 0;
14 var group: ShiftT = 0;
15
16 while (group < max_group) : (group += 1) {
17 const byte = try reader.readByte();
18
19 const ov = @shlWithOverflow(@as(U, byte & 0x7f), group * 7);
20 if (ov[1] != 0) return error.Overflow;
21
22 value |= ov[0];
23 if (byte & 0x80 == 0) break;
24 } else {
25 return error.Overflow;
26 }
27
28 // only applies in the case that we extended to u8
29 if (U != T) {
30 if (value > std.math.maxInt(T)) return error.Overflow;
31 }
32
33 return @as(T, @truncate(value));
34}
35
36/// Deprecated: use `readUleb128`
37pub const readULEB128 = readUleb128;
38
39/// Write a single unsigned integer as unsigned LEB128 to the given writer.
40pub fn writeUleb128(writer: anytype, arg: anytype) !void {
41 const Arg = @TypeOf(arg);
42 const Int = switch (Arg) {
43 comptime_int => std.math.IntFittingRange(arg, arg),
44 else => Arg,
45 };
46 const Value = if (@typeInfo(Int).int.bits < 8) u8 else Int;
47 var value: Value = arg;
48
49 while (true) {
50 const byte: u8 = @truncate(value & 0x7f);
51 value >>= 7;
52 if (value == 0) {
53 try writer.writeByte(byte);
54 break;
55 } else {
56 try writer.writeByte(byte | 0x80);
57 }
58 }
59}
60
61/// Deprecated: use `writeUleb128`
62pub const writeULEB128 = writeUleb128;
63
64/// Read a single signed LEB128 value from the given reader as type T,
65/// or error.Overflow if the value cannot fit.
66pub fn readIleb128(comptime T: type, reader: anytype) !T {
67 const S = if (@typeInfo(T).int.bits < 8) i8 else T;
68 const U = std.meta.Int(.unsigned, @typeInfo(S).int.bits);
69 const ShiftU = std.math.Log2Int(U);
70
71 const max_group = (@typeInfo(U).int.bits + 6) / 7;
72
73 var value = @as(U, 0);
74 var group = @as(ShiftU, 0);
75
76 while (group < max_group) : (group += 1) {
77 const byte = try reader.readByte();
78
79 const shift = group * 7;
80 const ov = @shlWithOverflow(@as(U, byte & 0x7f), shift);
81 if (ov[1] != 0) {
82 // Overflow is ok so long as the sign bit is set and this is the last byte
83 if (byte & 0x80 != 0) return error.Overflow;
84 if (@as(S, @bitCast(ov[0])) >= 0) return error.Overflow;
85
86 // and all the overflowed bits are 1
87 const remaining_shift = @as(u3, @intCast(@typeInfo(U).int.bits - @as(u16, shift)));
88 const remaining_bits = @as(i8, @bitCast(byte | 0x80)) >> remaining_shift;
89 if (remaining_bits != -1) return error.Overflow;
90 } else {
91 // If we don't overflow and this is the last byte and the number being decoded
92 // is negative, check that the remaining bits are 1
93 if ((byte & 0x80 == 0) and (@as(S, @bitCast(ov[0])) < 0)) {
94 const remaining_shift = @as(u3, @intCast(@typeInfo(U).int.bits - @as(u16, shift)));
95 const remaining_bits = @as(i8, @bitCast(byte | 0x80)) >> remaining_shift;
96 if (remaining_bits != -1) return error.Overflow;
97 }
98 }
99
100 value |= ov[0];
101 if (byte & 0x80 == 0) {
102 const needs_sign_ext = group + 1 < max_group;
103 if (byte & 0x40 != 0 and needs_sign_ext) {
104 const ones = @as(S, -1);
105 value |= @as(U, @bitCast(ones)) << (shift + 7);
106 }
107 break;
108 }
109 } else {
110 return error.Overflow;
111 }
112
113 const result = @as(S, @bitCast(value));
114 // Only applies if we extended to i8
115 if (S != T) {
116 if (result > std.math.maxInt(T) or result < std.math.minInt(T)) return error.Overflow;
117 }
118
119 return @as(T, @truncate(result));
120}
121
122/// Deprecated: use `readIleb128`
123pub const readILEB128 = readIleb128;
124
125/// Write a single signed integer as signed LEB128 to the given writer.
126pub fn writeIleb128(writer: anytype, arg: anytype) !void {
127 const Arg = @TypeOf(arg);
128 const Int = switch (Arg) {
129 comptime_int => std.math.IntFittingRange(-@abs(arg), @abs(arg)),
130 else => Arg,
131 };
132 const Signed = if (@typeInfo(Int).int.bits < 8) i8 else Int;
133 const Unsigned = std.meta.Int(.unsigned, @typeInfo(Signed).int.bits);
134 var value: Signed = arg;
135
136 while (true) {
137 const unsigned: Unsigned = @bitCast(value);
138 const byte: u8 = @truncate(unsigned);
139 value >>= 6;
140 if (value == -1 or value == 0) {
141 try writer.writeByte(byte & 0x7F);
142 break;
143 } else {
144 value >>= 1;
145 try writer.writeByte(byte | 0x80);
146 }
147 }
148}
149
150/// This is an "advanced" function. It allows one to use a fixed amount of memory to store a5/// This is an "advanced" function. It allows one to use a fixed amount of memory to store a
151/// ULEB128. This defeats the entire purpose of using this data encoding; it will no longer use6/// ULEB128. This defeats the entire purpose of using this data encoding; it will no longer use
152/// fewer bytes to store smaller numbers. The advantage of using a fixed width is that it makes7/// fewer bytes to store smaller numbers. The advantage of using a fixed width is that it makes
...@@ -176,9 +31,6 @@ pub fn writeUnsignedExtended(slice: []u8, arg: anytype) void {...@@ -176,9 +31,6 @@ pub fn writeUnsignedExtended(slice: []u8, arg: anytype) void {
176 slice[slice.len - 1] = @as(u7, @intCast(value));31 slice[slice.len - 1] = @as(u7, @intCast(value));
177}32}
17833
179/// Deprecated: use `writeIleb128`
180pub const writeILEB128 = writeIleb128;
181
182test writeUnsignedFixed {34test writeUnsignedFixed {
183 {35 {
184 var buf: [4]u8 = undefined;36 var buf: [4]u8 = undefined;
...@@ -261,42 +113,45 @@ test writeSignedFixed {...@@ -261,42 +113,45 @@ test writeSignedFixed {
261 }113 }
262}114}
263115
264// tests
265fn test_read_stream_ileb128(comptime T: type, encoded: []const u8) !T {116fn test_read_stream_ileb128(comptime T: type, encoded: []const u8) !T {
266 var reader = std.io.fixedBufferStream(encoded);117 var br: std.io.BufferedReader = undefined;
267 return try readIleb128(T, reader.reader());118 br.initFixed(encoded);
119 return br.takeIleb128(T);
268}120}
269121
270fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {122fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T {
271 var reader = std.io.fixedBufferStream(encoded);123 var br: std.io.BufferedReader = undefined;
272 return try readUleb128(T, reader.reader());124 br.initFixed(encoded);
125 return br.takeUleb128(T);
273}126}
274127
275fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {128fn test_read_ileb128(comptime T: type, encoded: []const u8) !T {
276 var reader = std.io.fixedBufferStream(encoded);129 var br: std.io.BufferedReader = undefined;
277 const v1 = try readIleb128(T, reader.reader());130 br.initFixed(encoded);
278 return v1;131 return br.readIleb128(T);
279}132}
280133
281fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {134fn test_read_uleb128(comptime T: type, encoded: []const u8) !T {
282 var reader = std.io.fixedBufferStream(encoded);135 var br: std.io.BufferedReader = undefined;
283 const v1 = try readUleb128(T, reader.reader());136 br.initFixed(encoded);
284 return v1;137 return br.readUleb128(T);
285}138}
286139
287fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {140fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
288 var reader = std.io.fixedBufferStream(encoded);141 var br: std.io.BufferedReader = undefined;
142 br.initFixed(encoded);
289 var i: usize = 0;143 var i: usize = 0;
290 while (i < N) : (i += 1) {144 while (i < N) : (i += 1) {
291 _ = try readIleb128(T, reader.reader());145 _ = try br.readIleb128(T);
292 }146 }
293}147}
294148
295fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {149fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) !void {
296 var reader = std.io.fixedBufferStream(encoded);150 var br: std.io.BufferedReader = undefined;
151 br.initFixed(encoded);
297 var i: usize = 0;152 var i: usize = 0;
298 while (i < N) : (i += 1) {153 while (i < N) : (i += 1) {
299 _ = try readUleb128(T, reader.reader());154 _ = try br.readUleb128(T);
300 }155 }
301}156}
302157
...@@ -392,8 +247,8 @@ fn test_write_leb128(value: anytype) !void {...@@ -392,8 +247,8 @@ fn test_write_leb128(value: anytype) !void {
392 const signedness = @typeInfo(T).int.signedness;247 const signedness = @typeInfo(T).int.signedness;
393 const t_signed = signedness == .signed;248 const t_signed = signedness == .signed;
394249
395 const writeStream = if (t_signed) writeIleb128 else writeUleb128;250 const writeStream = if (t_signed) std.io.BufferedWriter.writeIleb128 else std.io.BufferedWriter.writeUleb128;
396 const readStream = if (t_signed) readIleb128 else readUleb128;251 const readStream = if (t_signed) std.io.BufferedReader.readIleb128 else std.io.BufferedReader.readUleb128;
397252
398 // decode to a larger bit size too, to ensure sign extension253 // decode to a larger bit size too, to ensure sign extension
399 // is working as expected254 // is working as expected
...@@ -412,23 +267,24 @@ fn test_write_leb128(value: anytype) !void {...@@ -412,23 +267,24 @@ fn test_write_leb128(value: anytype) !void {
412 const max_groups = if (@typeInfo(T).int.bits == 0) 1 else (@typeInfo(T).int.bits + 6) / 7;267 const max_groups = if (@typeInfo(T).int.bits == 0) 1 else (@typeInfo(T).int.bits + 6) / 7;
413268
414 var buf: [max_groups]u8 = undefined;269 var buf: [max_groups]u8 = undefined;
415 var fbs = std.io.fixedBufferStream(&buf);270 var bw: std.io.BufferedWriter = undefined;
271 bw.initFixed(&buf);
416272
417 // stream write273 // stream write
418 try writeStream(fbs.writer(), value);274 try testing.expect((try writeStream(&bw, value)) == bytes_needed);
419 const w1_pos = fbs.pos;275 try testing.expect(bw.buffer.items.len == bytes_needed);
420 try testing.expect(w1_pos == bytes_needed);
421276
422 // stream read277 // stream read
423 fbs.pos = 0;278 var br: std.io.BufferedReader = undefined;
424 const sr = try readStream(T, fbs.reader());279 br.initFixed(&buf);
425 try testing.expect(fbs.pos == w1_pos);280 const sr = try readStream(&br, T);
281 try testing.expect(br.seek == bytes_needed);
426 try testing.expect(sr == value);282 try testing.expect(sr == value);
427283
428 // bigger type stream read284 // bigger type stream read
429 fbs.pos = 0;285 bw.buffer.items.len = 0;
430 const bsr = try readStream(B, fbs.reader());286 const bsr = try readStream(&bw, B);
431 try testing.expect(fbs.pos == w1_pos);287 try testing.expect(bw.buffer.items.len == bytes_needed);
432 try testing.expect(bsr == value);288 try testing.expect(bsr == value);
433}289}
434290
lib/std/zig/ErrorBundle.zig+7-9
...@@ -189,24 +189,22 @@ fn renderErrorMessageToWriter(...@@ -189,24 +189,22 @@ fn renderErrorMessageToWriter(
189 indent: usize,189 indent: usize,
190) anyerror!void {190) anyerror!void {
191 const ttyconf = options.ttyconf;191 const ttyconf = options.ttyconf;
192 var counting_writer: std.io.CountingWriter = .{ .child_writer = bw.writer() };
193 var counting_bw = counting_writer.writer().unbuffered();
194 const err_msg = eb.getErrorMessage(err_msg_index);192 const err_msg = eb.getErrorMessage(err_msg_index);
193 // This is the length of the part before the error message:
194 // e.g. "file.zig:4:5: error: "
195 var prefix_len: usize = 0;
195 if (err_msg.src_loc != .none) {196 if (err_msg.src_loc != .none) {
196 const src = eb.extraData(SourceLocation, @intFromEnum(err_msg.src_loc));197 const src = eb.extraData(SourceLocation, @intFromEnum(err_msg.src_loc));
197 try counting_bw.splatByteAll(' ', indent);198 prefix_len += try bw.splatByteAllCount(' ', indent);
198 try ttyconf.setColor(bw, .bold);199 try ttyconf.setColor(bw, .bold);
199 try counting_bw.print("{s}:{d}:{d}: ", .{200 prefix_len += try bw.printCount("{s}:{d}:{d}: ", .{
200 eb.nullTerminatedString(src.data.src_path),201 eb.nullTerminatedString(src.data.src_path),
201 src.data.line + 1,202 src.data.line + 1,
202 src.data.column + 1,203 src.data.column + 1,
203 });204 });
204 try ttyconf.setColor(bw, color);205 try ttyconf.setColor(bw, color);
205 try counting_bw.writeAll(kind);206 prefix_len += try bw.writeAllCount(kind);
206 try counting_bw.writeAll(": ");207 prefix_len += try bw.writeAllCount(": ");
207 // This is the length of the part before the error message:
208 // e.g. "file.zig:4:5: error: "
209 const prefix_len: usize = @intCast(counting_writer.bytes_written);
210 try ttyconf.setColor(bw, .reset);208 try ttyconf.setColor(bw, .reset);
211 try ttyconf.setColor(bw, .bold);209 try ttyconf.setColor(bw, .bold);
212 if (err_msg.count == 1) {210 if (err_msg.count == 1) {
lib/std/zip/test.zig+151-152
...@@ -103,13 +103,13 @@ pub const Zip64Options = struct {...@@ -103,13 +103,13 @@ pub const Zip64Options = struct {
103};103};
104104
105pub fn writeZip(105pub fn writeZip(
106 writer: anytype,106 writer: *std.io.BufferedWriter,
107 files: []const File,107 files: []const File,
108 store: []FileStore,108 store: []FileStore,
109 options: WriteZipOptions,109 options: WriteZipOptions,
110) !void {110) !void {
111 if (store.len < files.len) return error.FileStoreTooSmall;111 if (store.len < files.len) return error.FileStoreTooSmall;
112 var zipper = initZipper(writer);112 var zipper: Zipper = .init(writer);
113 for (files, 0..) |file, i| {113 for (files, 0..) |file, i| {
114 store[i] = try zipper.writeFile(.{114 store[i] = try zipper.writeFile(.{
115 .name = file.name,115 .name = file.name,
...@@ -126,173 +126,172 @@ pub fn writeZip(...@@ -126,173 +126,172 @@ pub fn writeZip(
126 try zipper.writeEndRecord(if (options.end) |e| e else .{});126 try zipper.writeEndRecord(if (options.end) |e| e else .{});
127}127}
128128
129pub fn initZipper(writer: anytype) Zipper(@TypeOf(writer)) {
130 return .{ .counting_writer = std.io.countingWriter(writer) };
131}
132
133/// Provides methods to format and write the contents of a zip archive129/// Provides methods to format and write the contents of a zip archive
134/// to the underlying Writer.130/// to the underlying Writer.
135pub fn Zipper(comptime Writer: type) type {131pub const Zipper = struct {
136 return struct {132 writer: *std.io.BufferedWriter,
137 counting_writer: std.io.CountingWriter(Writer),133 bytes_written: u64,
138 central_count: u64 = 0,134 central_count: u64 = 0,
139 first_central_offset: ?u64 = null,135 first_central_offset: ?u64 = null,
140 last_central_limit: ?u64 = null,136 last_central_limit: ?u64 = null,
141137
142 const Self = @This();138 const Self = @This();
143139
144 pub fn writeFile(140 pub fn init(writer: *std.io.BufferedWriter) Zipper {
145 self: *Self,141 return .{ .writer = writer, .bytes_written = 0 };
146 opt: struct {142 }
147 name: []const u8,
148 content: []const u8,
149 compression: zip.CompressionMethod,
150 write_options: WriteZipOptions,
151 },
152 ) !FileStore {
153 const writer = self.counting_writer.writer();
154
155 const file_offset: u64 = @intCast(self.counting_writer.bytes_written);
156 const crc32 = std.hash.Crc32.hash(opt.content);
157143
158 const header_options = opt.write_options.local_header;144 pub fn writeFile(
159 {145 self: *Self,
160 var compressed_size: u32 = 0;146 opt: struct {
161 var uncompressed_size: u32 = 0;147 name: []const u8,
162 var extra_len: u16 = 0;148 content: []const u8,
163 if (header_options) |hdr_options| {149 compression: zip.CompressionMethod,
164 compressed_size = if (hdr_options.compressed_size) |size| size else 0;150 write_options: WriteZipOptions,
165 uncompressed_size = if (hdr_options.uncompressed_size) |size| size else @intCast(opt.content.len);151 },
166 extra_len = if (hdr_options.extra_len) |len| len else 0;152 ) !FileStore {
167 }153 const writer = self.writer;
168 const hdr: zip.LocalFileHeader = .{
169 .signature = zip.local_file_header_sig,
170 .version_needed_to_extract = 10,
171 .flags = .{ .encrypted = false, ._ = 0 },
172 .compression_method = opt.compression,
173 .last_modification_time = 0,
174 .last_modification_date = 0,
175 .crc32 = crc32,
176 .compressed_size = compressed_size,
177 .uncompressed_size = uncompressed_size,
178 .filename_len = @intCast(opt.name.len),
179 .extra_len = extra_len,
180 };
181 try writer.writeStructEndian(hdr, .little);
182 }
183 try writer.writeAll(opt.name);
184154
185 if (header_options) |hdr| {155 const file_offset: u64 = @intCast(self.bytes_written);
186 if (hdr.zip64) |options| {156 const crc32 = std.hash.Crc32.hash(opt.content);
187 try writer.writeInt(u16, 0x0001, .little);
188 const data_size = if (options.data_size) |size| size else 8;
189 try writer.writeInt(u16, data_size, .little);
190 try writer.writeInt(u64, 0, .little);
191 try writer.writeInt(u64, @intCast(opt.content.len), .little);
192 }
193 }
194157
195 var compressed_size: u32 = undefined;158 const header_options = opt.write_options.local_header;
196 switch (opt.compression) {159 {
197 .store => {160 var compressed_size: u32 = 0;
198 try writer.writeAll(opt.content);161 var uncompressed_size: u32 = 0;
199 compressed_size = @intCast(opt.content.len);162 var extra_len: u16 = 0;
200 },163 if (header_options) |hdr_options| {
201 .deflate => {164 compressed_size = if (hdr_options.compressed_size) |size| size else 0;
202 const offset = self.counting_writer.bytes_written;165 uncompressed_size = if (hdr_options.uncompressed_size) |size| size else @intCast(opt.content.len);
203 var fbs = std.io.fixedBufferStream(opt.content);166 extra_len = if (hdr_options.extra_len) |len| len else 0;
204 try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
205 std.debug.assert(fbs.pos == opt.content.len);
206 compressed_size = @intCast(self.counting_writer.bytes_written - offset);
207 },
208 else => unreachable,
209 }167 }
210 return .{168 const hdr: zip.LocalFileHeader = .{
211 .compression = opt.compression,169 .signature = zip.local_file_header_sig,
212 .file_offset = file_offset,170 .version_needed_to_extract = 10,
171 .flags = .{ .encrypted = false, ._ = 0 },
172 .compression_method = opt.compression,
173 .last_modification_time = 0,
174 .last_modification_date = 0,
213 .crc32 = crc32,175 .crc32 = crc32,
214 .compressed_size = compressed_size,176 .compressed_size = compressed_size,
215 .uncompressed_size = opt.content.len,177 .uncompressed_size = uncompressed_size,
178 .filename_len = @intCast(opt.name.len),
179 .extra_len = extra_len,
216 };180 };
181 self.bytes_written += try writer.writeStructEndian(hdr, .little);
217 }182 }
183 self.bytes_written += try writer.writeAll(opt.name);
218184
219 pub fn writeCentralRecord(185 if (header_options) |hdr| {
220 self: *Self,186 if (hdr.zip64) |options| {
221 store: FileStore,187 self.bytes_written += try writer.writeInt(u16, 0x0001, .little);
222 opt: struct {188 const data_size = if (options.data_size) |size| size else 8;
223 name: []const u8,189 self.bytes_written += try writer.writeInt(u16, data_size, .little);
224 version_needed_to_extract: u16 = 10,190 self.bytes_written += try writer.writeInt(u64, 0, .little);
225 },191 self.bytes_written += try writer.writeInt(u64, @intCast(opt.content.len), .little);
226 ) !void {
227 if (self.first_central_offset == null) {
228 self.first_central_offset = self.counting_writer.bytes_written;
229 }192 }
230 self.central_count += 1;193 }
231194
232 const hdr: zip.CentralDirectoryFileHeader = .{195 var compressed_size: u32 = undefined;
233 .signature = zip.central_file_header_sig,196 switch (opt.compression) {
234 .version_made_by = 0,197 .store => {
235 .version_needed_to_extract = opt.version_needed_to_extract,198 self.bytes_written += try writer.writeAll(opt.content);
236 .flags = .{ .encrypted = false, ._ = 0 },199 compressed_size = @intCast(opt.content.len);
237 .compression_method = store.compression,200 },
238 .last_modification_time = 0,201 .deflate => {
239 .last_modification_date = 0,202 const offset = self.bytes_written;
240 .crc32 = store.crc32,203 var fbs = std.io.fixedBufferStream(opt.content);
241 .compressed_size = store.compressed_size,204 self.bytes_written += try std.compress.flate.deflate.compress(.raw, fbs.reader(), writer, .{});
242 .uncompressed_size = @intCast(store.uncompressed_size),205 std.debug.assert(fbs.pos == opt.content.len);
243 .filename_len = @intCast(opt.name.len),206 compressed_size = @intCast(self.bytes_written - offset);
244 .extra_len = 0,207 },
245 .comment_len = 0,208 else => unreachable,
246 .disk_number = 0,
247 .internal_file_attributes = 0,
248 .external_file_attributes = 0,
249 .local_file_header_offset = @intCast(store.file_offset),
250 };
251 try self.counting_writer.writer().writeStructEndian(hdr, .little);
252 try self.counting_writer.writer().writeAll(opt.name);
253 self.last_central_limit = self.counting_writer.bytes_written;
254 }209 }
210 return .{
211 .compression = opt.compression,
212 .file_offset = file_offset,
213 .crc32 = crc32,
214 .compressed_size = compressed_size,
215 .uncompressed_size = opt.content.len,
216 };
217 }
255218
256 pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {219 pub fn writeCentralRecord(
257 const cd_offset = self.first_central_offset orelse 0;220 self: *Self,
258 const cd_end = self.last_central_limit orelse 0;221 store: FileStore,
222 opt: struct {
223 name: []const u8,
224 version_needed_to_extract: u16 = 10,
225 },
226 ) !void {
227 if (self.first_central_offset == null) {
228 self.first_central_offset = self.bytes_written;
229 }
230 self.central_count += 1;
259231
260 if (opt.zip64) |zip64| {232 const hdr: zip.CentralDirectoryFileHeader = .{
261 const end64_off = cd_end;233 .signature = zip.central_file_header_sig,
262 const fixed: zip.EndRecord64 = .{234 .version_made_by = 0,
263 .signature = zip.end_record64_sig,235 .version_needed_to_extract = opt.version_needed_to_extract,
264 .end_record_size = @sizeOf(zip.EndRecord64) - 12,236 .flags = .{ .encrypted = false, ._ = 0 },
265 .version_made_by = 0,237 .compression_method = store.compression,
266 .version_needed_to_extract = 45,238 .last_modification_time = 0,
267 .disk_number = 0,239 .last_modification_date = 0,
268 .central_directory_disk_number = 0,240 .crc32 = store.crc32,
269 .record_count_disk = @intCast(self.central_count),241 .compressed_size = store.compressed_size,
270 .record_count_total = @intCast(self.central_count),242 .uncompressed_size = @intCast(store.uncompressed_size),
271 .central_directory_size = @intCast(cd_end - cd_offset),243 .filename_len = @intCast(opt.name.len),
272 .central_directory_offset = @intCast(cd_offset),244 .extra_len = 0,
273 };245 .comment_len = 0,
274 try self.counting_writer.writer().writeStructEndian(fixed, .little);246 .disk_number = 0,
275 const locator: zip.EndLocator64 = .{247 .internal_file_attributes = 0,
276 .signature = if (zip64.locator_sig) |s| s else zip.end_locator64_sig,248 .external_file_attributes = 0,
277 .zip64_disk_count = if (zip64.locator_zip64_disk_count) |c| c else 0,249 .local_file_header_offset = @intCast(store.file_offset),
278 .record_file_offset = if (zip64.locator_record_file_offset) |o| o else @intCast(end64_off),250 };
279 .total_disk_count = if (zip64.locator_total_disk_count) |c| c else 1,251 self.bytes_written += try self.writer.writeStructEndian(hdr, .little);
280 };252 self.bytes_written += try self.writer.writeAll(opt.name);
281 try self.counting_writer.writer().writeStructEndian(locator, .little);253 self.last_central_limit = self.bytes_written;
282 }254 }
283 const hdr: zip.EndRecord = .{255
284 .signature = if (opt.sig) |s| s else zip.end_record_sig,256 pub fn writeEndRecord(self: *Self, opt: EndRecordOptions) !void {
285 .disk_number = if (opt.disk_number) |n| n else 0,257 const cd_offset = self.first_central_offset orelse 0;
286 .central_directory_disk_number = if (opt.central_directory_disk_number) |n| n else 0,258 const cd_end = self.last_central_limit orelse 0;
287 .record_count_disk = if (opt.record_count_disk) |c| c else @intCast(self.central_count),259
288 .record_count_total = if (opt.record_count_total) |c| c else @intCast(self.central_count),260 if (opt.zip64) |zip64| {
289 .central_directory_size = if (opt.central_directory_size) |s| s else @intCast(cd_end - cd_offset),261 const end64_off = cd_end;
290 .central_directory_offset = if (opt.central_directory_offset) |o| o else @intCast(cd_offset),262 const fixed: zip.EndRecord64 = .{
291 .comment_len = if (opt.comment_len) |l| l else (if (opt.comment) |c| @as(u16, @intCast(c.len)) else 0),263 .signature = zip.end_record64_sig,
264 .end_record_size = @sizeOf(zip.EndRecord64) - 12,
265 .version_made_by = 0,
266 .version_needed_to_extract = 45,
267 .disk_number = 0,
268 .central_directory_disk_number = 0,
269 .record_count_disk = @intCast(self.central_count),
270 .record_count_total = @intCast(self.central_count),
271 .central_directory_size = @intCast(cd_end - cd_offset),
272 .central_directory_offset = @intCast(cd_offset),
273 };
274 self.bytes_written += try self.writer.writeStructEndian(fixed, .little);
275 const locator: zip.EndLocator64 = .{
276 .signature = if (zip64.locator_sig) |s| s else zip.end_locator64_sig,
277 .zip64_disk_count = if (zip64.locator_zip64_disk_count) |c| c else 0,
278 .record_file_offset = if (zip64.locator_record_file_offset) |o| o else @intCast(end64_off),
279 .total_disk_count = if (zip64.locator_total_disk_count) |c| c else 1,
292 };280 };
293 try self.counting_writer.writer().writeStructEndian(hdr, .little);281 self.bytes_written += try self.writer.writeStructEndian(locator, .little);
294 if (opt.comment) |c|
295 try self.counting_writer.writer().writeAll(c);
296 }282 }
297 };283 const hdr: zip.EndRecord = .{
298}284 .signature = if (opt.sig) |s| s else zip.end_record_sig,
285 .disk_number = if (opt.disk_number) |n| n else 0,
286 .central_directory_disk_number = if (opt.central_directory_disk_number) |n| n else 0,
287 .record_count_disk = if (opt.record_count_disk) |c| c else @intCast(self.central_count),
288 .record_count_total = if (opt.record_count_total) |c| c else @intCast(self.central_count),
289 .central_directory_size = if (opt.central_directory_size) |s| s else @intCast(cd_end - cd_offset),
290 .central_directory_offset = if (opt.central_directory_offset) |o| o else @intCast(cd_offset),
291 .comment_len = if (opt.comment_len) |l| l else (if (opt.comment) |c| @as(u16, @intCast(c.len)) else 0),
292 };
293 self.bytes_written += try self.writer.writeStructEndian(hdr, .little);
294 if (opt.comment) |c|
295 self.bytes_written += try self.writer.writeAll(c);
296 }
297};
src/Type.zig+92-85
...@@ -121,11 +121,10 @@ pub fn eql(a: Type, b: Type, zcu: *const Zcu) bool {...@@ -121,11 +121,10 @@ pub fn eql(a: Type, b: Type, zcu: *const Zcu) bool {
121 return a.toIntern() == b.toIntern();121 return a.toIntern() == b.toIntern();
122}122}
123123
124pub fn format(ty: Type, comptime unused_fmt_string: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void {124pub fn format(ty: Type, bw: *std.io.BufferedWriter, comptime f: []const u8) anyerror!usize {
125 _ = ty;125 _ = ty;
126 _ = unused_fmt_string;126 _ = f;
127 _ = options;127 _ = bw;
128 _ = writer;
129 @compileError("do not format types directly; use either ty.fmtDebug() or ty.fmt()");128 @compileError("do not format types directly; use either ty.fmtDebug() or ty.fmt()");
130}129}
131130
...@@ -143,15 +142,9 @@ const FormatContext = struct {...@@ -143,15 +142,9 @@ const FormatContext = struct {
143 pt: Zcu.PerThread,142 pt: Zcu.PerThread,
144};143};
145144
146fn format2(145fn format2(ctx: FormatContext, bw: *std.io.BufferedWriter, comptime f: []const u8) anyerror!usize {
147 ctx: FormatContext,146 comptime assert(f.len == 0);
148 comptime unused_format_string: []const u8,147 return print(ctx.ty, bw, ctx.pt);
149 options: std.fmt.FormatOptions,
150 writer: anytype,
151) !void {
152 comptime assert(unused_format_string.len == 0);
153 _ = options;
154 return print(ctx.ty, writer, ctx.pt);
155}148}
156149
157pub fn fmtDebug(ty: Type) std.fmt.Formatter(dump) {150pub fn fmtDebug(ty: Type) std.fmt.Formatter(dump) {
...@@ -173,7 +166,7 @@ pub fn dump(...@@ -173,7 +166,7 @@ pub fn dump(
173166
174/// Prints a name suitable for `@typeName`.167/// Prints a name suitable for `@typeName`.
175/// TODO: take an `opt_sema` to pass to `fmtValue` when printing sentinels.168/// TODO: take an `opt_sema` to pass to `fmtValue` when printing sentinels.
176pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerror!void {169pub fn print(ty: Type, bw: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerror!usize {
177 const zcu = pt.zcu;170 const zcu = pt.zcu;
178 const ip = &zcu.intern_pool;171 const ip = &zcu.intern_pool;
179 switch (ip.indexToKey(ty.toIntern())) {172 switch (ip.indexToKey(ty.toIntern())) {
...@@ -183,22 +176,23 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr...@@ -183,22 +176,23 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr
183 .signed => 'i',176 .signed => 'i',
184 .unsigned => 'u',177 .unsigned => 'u',
185 };178 };
186 return writer.print("{c}{d}", .{ sign_char, int_type.bits });179 return bw.print("{c}{d}", .{ sign_char, int_type.bits });
187 },180 },
188 .ptr_type => {181 .ptr_type => {
182 var n: usize = 0;
189 const info = ty.ptrInfo(zcu);183 const info = ty.ptrInfo(zcu);
190184
191 if (info.sentinel != .none) switch (info.flags.size) {185 if (info.sentinel != .none) switch (info.flags.size) {
192 .one, .c => unreachable,186 .one, .c => unreachable,
193 .many => try writer.print("[*:{}]", .{Value.fromInterned(info.sentinel).fmtValue(pt)}),187 .many => n += try bw.print("[*:{}]", .{Value.fromInterned(info.sentinel).fmtValue(pt)}),
194 .slice => try writer.print("[:{}]", .{Value.fromInterned(info.sentinel).fmtValue(pt)}),188 .slice => n += try bw.print("[:{}]", .{Value.fromInterned(info.sentinel).fmtValue(pt)}),
195 } else switch (info.flags.size) {189 } else switch (info.flags.size) {
196 .one => try writer.writeAll("*"),190 .one => n += try bw.writeAll("*"),
197 .many => try writer.writeAll("[*]"),191 .many => n += try bw.writeAll("[*]"),
198 .c => try writer.writeAll("[*c]"),192 .c => n += try bw.writeAll("[*c]"),
199 .slice => try writer.writeAll("[]"),193 .slice => n += try bw.writeAll("[]"),
200 }194 }
201 if (info.flags.is_allowzero and info.flags.size != .c) try writer.writeAll("allowzero ");195 if (info.flags.is_allowzero and info.flags.size != .c) n += try bw.writeAll("allowzero ");
202 if (info.flags.alignment != .none or196 if (info.flags.alignment != .none or
203 info.packed_offset.host_size != 0 or197 info.packed_offset.host_size != 0 or
204 info.flags.vector_index != .none)198 info.flags.vector_index != .none)
...@@ -207,76 +201,83 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr...@@ -207,76 +201,83 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr
207 info.flags.alignment201 info.flags.alignment
208 else202 else
209 Type.fromInterned(info.child).abiAlignment(pt.zcu);203 Type.fromInterned(info.child).abiAlignment(pt.zcu);
210 try writer.print("align({d}", .{alignment.toByteUnits() orelse 0});204 n += try bw.print("align({d}", .{alignment.toByteUnits() orelse 0});
211205
212 if (info.packed_offset.bit_offset != 0 or info.packed_offset.host_size != 0) {206 if (info.packed_offset.bit_offset != 0 or info.packed_offset.host_size != 0) {
213 try writer.print(":{d}:{d}", .{207 n += try bw.print(":{d}:{d}", .{
214 info.packed_offset.bit_offset, info.packed_offset.host_size,208 info.packed_offset.bit_offset, info.packed_offset.host_size,
215 });209 });
216 }210 }
217 if (info.flags.vector_index == .runtime) {211 if (info.flags.vector_index == .runtime) {
218 try writer.writeAll(":?");212 n += try bw.writeAll(":?");
219 } else if (info.flags.vector_index != .none) {213 } else if (info.flags.vector_index != .none) {
220 try writer.print(":{d}", .{@intFromEnum(info.flags.vector_index)});214 n += try bw.print(":{d}", .{@intFromEnum(info.flags.vector_index)});
221 }215 }
222 try writer.writeAll(") ");216 n += try bw.writeAll(") ");
223 }217 }
224 if (info.flags.address_space != .generic) {218 if (info.flags.address_space != .generic) {
225 try writer.print("addrspace(.{s}) ", .{@tagName(info.flags.address_space)});219 n += try bw.print("addrspace(.{s}) ", .{@tagName(info.flags.address_space)});
226 }220 }
227 if (info.flags.is_const) try writer.writeAll("const ");221 if (info.flags.is_const) n += try bw.writeAll("const ");
228 if (info.flags.is_volatile) try writer.writeAll("volatile ");222 if (info.flags.is_volatile) n += try bw.writeAll("volatile ");
229223
230 try print(Type.fromInterned(info.child), writer, pt);224 n += try print(Type.fromInterned(info.child), bw, pt);
231 return;225 return n;
232 },226 },
233 .array_type => |array_type| {227 .array_type => |array_type| {
228 var n: usize = 0;
234 if (array_type.sentinel == .none) {229 if (array_type.sentinel == .none) {
235 try writer.print("[{d}]", .{array_type.len});230 n += try bw.print("[{d}]", .{array_type.len});
236 try print(Type.fromInterned(array_type.child), writer, pt);231 n += try print(Type.fromInterned(array_type.child), bw, pt);
237 } else {232 } else {
238 try writer.print("[{d}:{}]", .{233 n += try bw.print("[{d}:{}]", .{
239 array_type.len,234 array_type.len,
240 Value.fromInterned(array_type.sentinel).fmtValue(pt),235 Value.fromInterned(array_type.sentinel).fmtValue(pt),
241 });236 });
242 try print(Type.fromInterned(array_type.child), writer, pt);237 n += try print(Type.fromInterned(array_type.child), bw, pt);
243 }238 }
244 return;239 return n;
245 },240 },
246 .vector_type => |vector_type| {241 .vector_type => |vector_type| {
247 try writer.print("@Vector({d}, ", .{vector_type.len});242 var n: usize = 0;
248 try print(Type.fromInterned(vector_type.child), writer, pt);243 n += try bw.print("@Vector({d}, ", .{vector_type.len});
249 try writer.writeAll(")");244 n += try print(Type.fromInterned(vector_type.child), bw, pt);
250 return;245 n += try bw.writeAll(")");
246 return n;
251 },247 },
252 .opt_type => |child| {248 .opt_type => |child| {
253 try writer.writeByte('?');249 var n: usize = 0;
254 return print(Type.fromInterned(child), writer, pt);250 n += try bw.writeByte('?');
251 n += try print(Type.fromInterned(child), bw, pt);
252 return n;
255 },253 },
256 .error_union_type => |error_union_type| {254 .error_union_type => |error_union_type| {
257 try print(Type.fromInterned(error_union_type.error_set_type), writer, pt);255 var n: usize = 0;
258 try writer.writeByte('!');256 n += try print(Type.fromInterned(error_union_type.error_set_type), bw, pt);
257 n += try bw.writeByte('!');
259 if (error_union_type.payload_type == .generic_poison_type) {258 if (error_union_type.payload_type == .generic_poison_type) {
260 try writer.writeAll("anytype");259 n += try bw.writeAll("anytype");
261 } else {260 } else {
262 try print(Type.fromInterned(error_union_type.payload_type), writer, pt);261 n += try print(Type.fromInterned(error_union_type.payload_type), bw, pt);
263 }262 }
264 return;263 return n;
265 },264 },
266 .inferred_error_set_type => |func_index| {265 .inferred_error_set_type => |func_index| {
267 const func_nav = ip.getNav(zcu.funcInfo(func_index).owner_nav);266 const func_nav = ip.getNav(zcu.funcInfo(func_index).owner_nav);
268 try writer.print("@typeInfo(@typeInfo(@TypeOf({})).@\"fn\".return_type.?).error_union.error_set", .{267 return bw.print("@typeInfo(@typeInfo(@TypeOf({})).@\"fn\".return_type.?).error_union.error_set", .{
269 func_nav.fqn.fmt(ip),268 func_nav.fqn.fmt(ip),
270 });269 });
271 },270 },
272 .error_set_type => |error_set_type| {271 .error_set_type => |error_set_type| {
272 var n: usize = 0;
273 const names = error_set_type.names;273 const names = error_set_type.names;
274 try writer.writeAll("error{");274 n += try bw.writeAll("error{");
275 for (names.get(ip), 0..) |name, i| {275 for (names.get(ip), 0..) |name, i| {
276 if (i != 0) try writer.writeByte(',');276 if (i != 0) n += try bw.writeByte(',');
277 try writer.print("{}", .{name.fmt(ip)});277 n += try bw.print("{}", .{name.fmt(ip)});
278 }278 }
279 try writer.writeAll("}");279 n += try bw.writeAll("}");
280 return n;
280 },281 },
281 .simple_type => |s| switch (s) {282 .simple_type => |s| switch (s) {
282 .f16,283 .f16,
...@@ -305,97 +306,103 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr...@@ -305,97 +306,103 @@ pub fn print(ty: Type, writer: *std.io.BufferedWriter, pt: Zcu.PerThread) anyerr
305 .comptime_float,306 .comptime_float,
306 .noreturn,307 .noreturn,
307 .adhoc_inferred_error_set,308 .adhoc_inferred_error_set,
308 => return writer.writeAll(@tagName(s)),309 => return bw.writeAll(@tagName(s)),
309310
310 .null,311 .null,
311 .undefined,312 .undefined,
312 => try writer.print("@TypeOf({s})", .{@tagName(s)}),313 => return bw.print("@TypeOf({s})", .{@tagName(s)}),
313314
314 .enum_literal => try writer.writeAll("@Type(.enum_literal)"),315 .enum_literal => return bw.writeAll("@Type(.enum_literal)"),
315316
316 .generic_poison => unreachable,317 .generic_poison => unreachable,
317 },318 },
318 .struct_type => {319 .struct_type => {
319 const name = ip.loadStructType(ty.toIntern()).name;320 const name = ip.loadStructType(ty.toIntern()).name;
320 try writer.print("{}", .{name.fmt(ip)});321 return bw.print("{}", .{name.fmt(ip)});
321 },322 },
322 .tuple_type => |tuple| {323 .tuple_type => |tuple| {
323 if (tuple.types.len == 0) {324 if (tuple.types.len == 0) {
324 return writer.writeAll("@TypeOf(.{})");325 return bw.writeAll("@TypeOf(.{})");
325 }326 }
326 try writer.writeAll("struct {");327 var n: usize = 0;
328 n += try bw.writeAll("struct {");
327 for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, val, i| {329 for (tuple.types.get(ip), tuple.values.get(ip), 0..) |field_ty, val, i| {
328 try writer.writeAll(if (i == 0) " " else ", ");330 n += try bw.writeAll(if (i == 0) " " else ", ");
329 if (val != .none) try writer.writeAll("comptime ");331 if (val != .none) n += try bw.writeAll("comptime ");
330 try print(Type.fromInterned(field_ty), writer, pt);332 n += try print(Type.fromInterned(field_ty), bw, pt);
331 if (val != .none) try writer.print(" = {}", .{Value.fromInterned(val).fmtValue(pt)});333 if (val != .none) n += try bw.print(" = {}", .{Value.fromInterned(val).fmtValue(pt)});
332 }334 }
333 try writer.writeAll(" }");335 n += try bw.writeAll(" }");
336 return n;
334 },337 },
335338
336 .union_type => {339 .union_type => {
337 const name = ip.loadUnionType(ty.toIntern()).name;340 const name = ip.loadUnionType(ty.toIntern()).name;
338 try writer.print("{}", .{name.fmt(ip)});341 return bw.print("{}", .{name.fmt(ip)});
339 },342 },
340 .opaque_type => {343 .opaque_type => {
341 const name = ip.loadOpaqueType(ty.toIntern()).name;344 const name = ip.loadOpaqueType(ty.toIntern()).name;
342 try writer.print("{}", .{name.fmt(ip)});345 return bw.print("{}", .{name.fmt(ip)});
343 },346 },
344 .enum_type => {347 .enum_type => {
345 const name = ip.loadEnumType(ty.toIntern()).name;348 const name = ip.loadEnumType(ty.toIntern()).name;
346 try writer.print("{}", .{name.fmt(ip)});349 return bw.print("{}", .{name.fmt(ip)});
347 },350 },
348 .func_type => |fn_info| {351 .func_type => |fn_info| {
352 var n: usize = 0;
349 if (fn_info.is_noinline) {353 if (fn_info.is_noinline) {
350 try writer.writeAll("noinline ");354 n += try bw.writeAll("noinline ");
351 }355 }
352 try writer.writeAll("fn (");356 n += try bw.writeAll("fn (");
353 const param_types = fn_info.param_types.get(&zcu.intern_pool);357 const param_types = fn_info.param_types.get(&zcu.intern_pool);
354 for (param_types, 0..) |param_ty, i| {358 for (param_types, 0..) |param_ty, i| {
355 if (i != 0) try writer.writeAll(", ");359 if (i != 0) n += try bw.writeAll(", ");
356 if (std.math.cast(u5, i)) |index| {360 if (std.math.cast(u5, i)) |index| {
357 if (fn_info.paramIsComptime(index)) {361 if (fn_info.paramIsComptime(index)) {
358 try writer.writeAll("comptime ");362 n += try bw.writeAll("comptime ");
359 }363 }
360 if (fn_info.paramIsNoalias(index)) {364 if (fn_info.paramIsNoalias(index)) {
361 try writer.writeAll("noalias ");365 n += try bw.writeAll("noalias ");
362 }366 }
363 }367 }
364 if (param_ty == .generic_poison_type) {368 if (param_ty == .generic_poison_type) {
365 try writer.writeAll("anytype");369 n += try bw.writeAll("anytype");
366 } else {370 } else {
367 try print(Type.fromInterned(param_ty), writer, pt);371 n += try print(Type.fromInterned(param_ty), bw, pt);
368 }372 }
369 }373 }
370 if (fn_info.is_var_args) {374 if (fn_info.is_var_args) {
371 if (param_types.len != 0) {375 if (param_types.len != 0) {
372 try writer.writeAll(", ");376 n += try bw.writeAll(", ");
373 }377 }
374 try writer.writeAll("...");378 n += try bw.writeAll("...");
375 }379 }
376 try writer.writeAll(") ");380 n += try bw.writeAll(") ");
377 if (fn_info.cc != .auto) print_cc: {381 if (fn_info.cc != .auto) print_cc: {
378 if (zcu.getTarget().cCallingConvention()) |ccc| {382 if (zcu.getTarget().cCallingConvention()) |ccc| {
379 if (fn_info.cc.eql(ccc)) {383 if (fn_info.cc.eql(ccc)) {
380 try writer.writeAll("callconv(.c) ");384 n += try bw.writeAll("callconv(.c) ");
381 break :print_cc;385 break :print_cc;
382 }386 }
383 }387 }
384 switch (fn_info.cc) {388 switch (fn_info.cc) {
385 .auto, .@"async", .naked, .@"inline" => try writer.print("callconv(.{}) ", .{std.zig.fmtId(@tagName(fn_info.cc))}),389 .auto, .@"async", .naked, .@"inline" => n += try bw.print("callconv(.{}) ", .{std.zig.fmtId(@tagName(fn_info.cc))}),
386 else => try writer.print("callconv({any}) ", .{fn_info.cc}),390 else => n += try bw.print("callconv({any}) ", .{fn_info.cc}),
387 }391 }
388 }392 }
389 if (fn_info.return_type == .generic_poison_type) {393 if (fn_info.return_type == .generic_poison_type) {
390 try writer.writeAll("anytype");394 n += try bw.writeAll("anytype");
391 } else {395 } else {
392 try print(Type.fromInterned(fn_info.return_type), writer, pt);396 n += try print(Type.fromInterned(fn_info.return_type), bw, pt);
393 }397 }
398 return n;
394 },399 },
395 .anyframe_type => |child| {400 .anyframe_type => |child| {
396 if (child == .none) return writer.writeAll("anyframe");401 if (child == .none) return bw.writeAll("anyframe");
397 try writer.writeAll("anyframe->");402 var n: usize = 0;
398 return print(Type.fromInterned(child), writer, pt);403 n += try bw.writeAll("anyframe->");
404 n += print(Type.fromInterned(child), bw, pt);
405 return n;
399 },406 },
400407
401 // values, not types408 // values, not types
src/codegen/c.zig+61-69
...@@ -1270,7 +1270,7 @@ pub const DeclGen = struct {...@@ -1270,7 +1270,7 @@ pub const DeclGen = struct {
1270 }1270 }
1271 const ai = ty.arrayInfo(zcu);1271 const ai = ty.arrayInfo(zcu);
1272 if (ai.elem_type.eql(.u8, zcu)) {1272 if (ai.elem_type.eql(.u8, zcu)) {
1273 var literal = stringLiteral(writer, ty.arrayLenIncludingSentinel(zcu));1273 var literal: StringLiteral = .init(writer, ty.arrayLenIncludingSentinel(zcu));
1274 try literal.start();1274 try literal.start();
1275 var index: usize = 0;1275 var index: usize = 0;
1276 while (index < ai.len) : (index += 1) {1276 while (index < ai.len) : (index += 1) {
...@@ -1829,7 +1829,7 @@ pub const DeclGen = struct {...@@ -1829,7 +1829,7 @@ pub const DeclGen = struct {
1829 const ai = ty.arrayInfo(zcu);1829 const ai = ty.arrayInfo(zcu);
1830 if (ai.elem_type.eql(.u8, zcu)) {1830 if (ai.elem_type.eql(.u8, zcu)) {
1831 const c_len = ty.arrayLenIncludingSentinel(zcu);1831 const c_len = ty.arrayLenIncludingSentinel(zcu);
1832 var literal = stringLiteral(writer, c_len);1832 var literal: StringLiteral = .init(writer, c_len);
1833 try literal.start();1833 try literal.start();
1834 var index: u64 = 0;1834 var index: u64 = 0;
1835 while (index < c_len) : (index += 1)1835 while (index < c_len) : (index += 1)
...@@ -8111,7 +8111,12 @@ fn compareOperatorC(operator: std.math.CompareOperator) []const u8 {...@@ -8111,7 +8111,12 @@ fn compareOperatorC(operator: std.math.CompareOperator) []const u8 {
8111 };8111 };
8112}8112}
81138113
8114fn StringLiteral(comptime WriterType: type) type {8114const StringLiteral = struct {
8115 len: usize,
8116 cur_len: usize,
8117 bytes_written: usize,
8118 writer: *std.io.BufferedWriter,
8119
8115 // MSVC throws C2078 if an array of size 65536 or greater is initialized with a string literal,8120 // MSVC throws C2078 if an array of size 65536 or greater is initialized with a string literal,
8116 // regardless of the length of the string literal initializing it. Array initializer syntax is8121 // regardless of the length of the string literal initializing it. Array initializer syntax is
8117 // used instead.8122 // used instead.
...@@ -8123,81 +8128,68 @@ fn StringLiteral(comptime WriterType: type) type {...@@ -8123,81 +8128,68 @@ fn StringLiteral(comptime WriterType: type) type {
8123 const max_char_len = 4;8128 const max_char_len = 4;
8124 const max_literal_len = @min(16380 - max_char_len, 4095);8129 const max_literal_len = @min(16380 - max_char_len, 4095);
81258130
8126 return struct {8131 fn init(writer: *std.io.BufferedWriter, len: usize) StringLiteral {
8127 len: u64,8132 return .{
8128 cur_len: u64 = 0,8133 .cur_len = 0,
8129 counting_writer: std.io.CountingWriter(WriterType),8134 .len = len,
81308135 .writer = writer,
8131 pub const Error = WriterType.Error;8136 .bytes_written = 0,
81328137 };
8133 const Self = @This();8138 }
81348139
8135 pub fn start(self: *Self) Error!void {8140 pub fn start(self: *StringLiteral) anyerror!void {
8136 const writer = self.counting_writer.writer();8141 const writer = self.writer;
8137 if (self.len <= max_string_initializer_len) {8142 if (self.len <= max_string_initializer_len) {
8138 try writer.writeByte('\"');8143 self.bytes_written += try writer.writeByteCount('\"');
8139 } else {8144 } else {
8140 try writer.writeByte('{');8145 self.bytes_written += try writer.writeByteCount('{');
8141 }
8142 }8146 }
8147 }
81438148
8144 pub fn end(self: *Self) Error!void {8149 pub fn end(self: *StringLiteral) anyerror!void {
8145 const writer = self.counting_writer.writer();8150 const writer = self.writer;
8146 if (self.len <= max_string_initializer_len) {8151 if (self.len <= max_string_initializer_len) {
8147 try writer.writeByte('\"');8152 self.bytes_written += try writer.writeByteCount('\"');
8148 } else {8153 } else {
8149 try writer.writeByte('}');8154 self.bytes_written += try writer.writeByteCount('}');
8150 }
8151 }8155 }
8156 }
81528157
8153 fn writeStringLiteralChar(writer: anytype, c: u8) !void {8158 fn writeStringLiteralChar(writer: *std.io.BufferedWriter, c: u8) anyerror!usize {
8154 switch (c) {8159 switch (c) {
8155 7 => try writer.writeAll("\\a"),8160 7 => return writer.writeAllCount("\\a"),
8156 8 => try writer.writeAll("\\b"),8161 8 => return writer.writeAllCount("\\b"),
8157 '\t' => try writer.writeAll("\\t"),8162 '\t' => return writer.writeAllCount("\\t"),
8158 '\n' => try writer.writeAll("\\n"),8163 '\n' => return writer.writeAllCount("\\n"),
8159 11 => try writer.writeAll("\\v"),8164 11 => return writer.writeAllCount("\\v"),
8160 12 => try writer.writeAll("\\f"),8165 12 => return writer.writeAllCount("\\f"),
8161 '\r' => try writer.writeAll("\\r"),8166 '\r' => return writer.writeAllCount("\\r"),
8162 '"', '\'', '?', '\\' => try writer.print("\\{c}", .{c}),8167 '"', '\'', '?', '\\' => return writer.printCount("\\{c}", .{c}),
8163 else => switch (c) {8168 else => switch (c) {
8164 ' '...'~' => try writer.writeByte(c),8169 ' '...'~' => return writer.writeByteCount(c),
8165 else => try writer.print("\\{o:0>3}", .{c}),8170 else => return writer.printCount("\\{o:0>3}", .{c}),
8166 },8171 },
8167 }
8168 }8172 }
8173 }
81698174
8170 pub fn writeChar(self: *Self, c: u8) Error!void {8175 pub fn writeChar(self: *StringLiteral, c: u8) anyerror!void {
8171 const writer = self.counting_writer.writer();8176 const writer = self.writer;
8172 if (self.len <= max_string_initializer_len) {8177 if (self.len <= max_string_initializer_len) {
8173 if (self.cur_len == 0 and self.counting_writer.bytes_written > 1)8178 if (self.cur_len == 0 and self.bytes_written > 1)
8174 try writer.writeAll("\"\"");8179 self.bytes_written += try writer.writeAllCount("\"\"");
8175
8176 const len = self.counting_writer.bytes_written;
8177 try writeStringLiteralChar(writer, c);
81788180
8179 const char_length = self.counting_writer.bytes_written - len;8181 const char_length = try writeStringLiteralChar(writer, c);
8180 assert(char_length <= max_char_len);8182 self.bytes_written += char_length;
8181 self.cur_len += char_length;8183 assert(char_length <= max_char_len);
8184 self.cur_len += char_length;
81828185
8183 if (self.cur_len >= max_literal_len) self.cur_len = 0;8186 if (self.cur_len >= max_literal_len) self.cur_len = 0;
8184 } else {8187 } else {
8185 if (self.counting_writer.bytes_written > 1) try writer.writeByte(',');8188 if (self.bytes_written > 1) self.bytes_written += try writer.writeByteCount(',');
8186 try writer.print("'\\x{x}'", .{c});8189 self.bytes_written += try writer.printCount("'\\x{x}'", .{c});
8187 }
8188 }8190 }
8189 };8191 }
8190}8192};
8191
8192fn stringLiteral(
8193 child_stream: anytype,
8194 len: u64,
8195) StringLiteral(@TypeOf(child_stream)) {
8196 return .{
8197 .len = len,
8198 .counting_writer = std.io.countingWriter(child_stream),
8199 };
8200}
82018193
8202const FormatStringContext = struct { str: []const u8, sentinel: ?u8 };8194const FormatStringContext = struct { str: []const u8, sentinel: ?u8 };
8203fn formatStringLiteral(8195fn formatStringLiteral(
...@@ -8208,7 +8200,7 @@ fn formatStringLiteral(...@@ -8208,7 +8200,7 @@ fn formatStringLiteral(
8208) @TypeOf(writer).Error!void {8200) @TypeOf(writer).Error!void {
8209 if (fmt.len != 1 or fmt[0] != 's') @compileError("Invalid fmt: " ++ fmt);8201 if (fmt.len != 1 or fmt[0] != 's') @compileError("Invalid fmt: " ++ fmt);
82108202
8211 var literal = stringLiteral(writer, data.str.len + @intFromBool(data.sentinel != null));8203 var literal: StringLiteral = .init(writer, data.str.len + @intFromBool(data.sentinel != null));
8212 try literal.start();8204 try literal.start();
8213 for (data.str) |c| try literal.writeChar(c);8205 for (data.str) |c| try literal.writeChar(c);
8214 if (data.sentinel) |sentinel| if (sentinel != 0) try literal.writeChar(sentinel);8206 if (data.sentinel) |sentinel| if (sentinel != 0) try literal.writeChar(sentinel);
src/link/Dwarf.zig+32-22
...@@ -1768,34 +1768,36 @@ pub const WipNav = struct {...@@ -1768,34 +1768,36 @@ pub const WipNav = struct {
1768 }1768 }
17691769
1770 const ExprLocCounter = struct {1770 const ExprLocCounter = struct {
1771 const Stream = std.io.CountingWriter(std.io.NullWriter);1771 stream: *std.io.BufferedWriter,
1772 stream: Stream,
1773 section_offset_bytes: u32,1772 section_offset_bytes: u32,
1774 address_size: AddressSize,1773 address_size: AddressSize,
1775 fn init(dwarf: *Dwarf) ExprLocCounter {1774 counter: usize,
1775 fn init(dwarf: *Dwarf, stream: *std.io.BufferedWriter) ExprLocCounter {
1776 return .{1776 return .{
1777 .stream = std.io.countingWriter(std.io.null_writer),1777 .stream = stream,
1778 .section_offset_bytes = dwarf.sectionOffsetBytes(),1778 .section_offset_bytes = dwarf.sectionOffsetBytes(),
1779 .address_size = dwarf.address_size,1779 .address_size = dwarf.address_size,
1780 };1780 };
1781 }1781 }
1782 fn writer(counter: *ExprLocCounter) Stream.Writer {1782 fn writer(counter: *ExprLocCounter) *std.io.BufferedWriter {
1783 return counter.stream.writer();1783 return counter.stream;
1784 }1784 }
1785 fn endian(_: ExprLocCounter) std.builtin.Endian {1785 fn endian(_: ExprLocCounter) std.builtin.Endian {
1786 return @import("builtin").cpu.arch.endian();1786 return @import("builtin").cpu.arch.endian();
1787 }1787 }
1788 fn addrSym(counter: *ExprLocCounter, _: u32) error{}!void {1788 fn addrSym(counter: *ExprLocCounter, _: u32) error{}!void {
1789 counter.stream.bytes_written += @intFromEnum(counter.address_size);1789 counter.count += @intFromEnum(counter.address_size);
1790 }1790 }
1791 fn infoEntry(counter: *ExprLocCounter, _: Unit.Index, _: Entry.Index) error{}!void {1791 fn infoEntry(counter: *ExprLocCounter, _: Unit.Index, _: Entry.Index) error{}!void {
1792 counter.stream.bytes_written += counter.section_offset_bytes;1792 counter.count += counter.section_offset_bytes;
1793 }1793 }
1794 };1794 };
17951795
1796 fn infoExprLoc(wip_nav: *WipNav, loc: Loc) UpdateError!void {1796 fn infoExprLoc(wip_nav: *WipNav, loc: Loc) UpdateError!void {
1797 var counter: ExprLocCounter = .init(wip_nav.dwarf);1797 var buffer: [std.atomic.cache_line]u8 = undefined;
1798 try loc.write(&counter);1798 var counter_bw = std.io.Writer.null.buffered(&buffer);
1799 var counter: ExprLocCounter = .init(wip_nav.dwarf, &counter_bw);
1800 counter.count += try loc.write(&counter);
17991801
1800 const adapter: struct {1802 const adapter: struct {
1801 wip_nav: *WipNav,1803 wip_nav: *WipNav,
...@@ -1812,8 +1814,8 @@ pub const WipNav = struct {...@@ -1812,8 +1814,8 @@ pub const WipNav = struct {
1812 try ctx.wip_nav.infoSectionOffset(.debug_info, unit, entry, 0);1814 try ctx.wip_nav.infoSectionOffset(.debug_info, unit, entry, 0);
1813 }1815 }
1814 } = .{ .wip_nav = wip_nav };1816 } = .{ .wip_nav = wip_nav };
1815 try uleb128(adapter.writer(), counter.stream.bytes_written);1817 try uleb128(adapter.writer(), counter.count);
1816 try loc.write(adapter);1818 _ = try loc.write(adapter);
1817 }1819 }
18181820
1819 fn infoAddrSym(wip_nav: *WipNav, sym_index: u32, sym_off: u64) UpdateError!void {1821 fn infoAddrSym(wip_nav: *WipNav, sym_index: u32, sym_off: u64) UpdateError!void {
...@@ -1826,8 +1828,10 @@ pub const WipNav = struct {...@@ -1826,8 +1828,10 @@ pub const WipNav = struct {
1826 }1828 }
18271829
1828 fn frameExprLoc(wip_nav: *WipNav, loc: Loc) UpdateError!void {1830 fn frameExprLoc(wip_nav: *WipNav, loc: Loc) UpdateError!void {
1829 var counter: ExprLocCounter = .init(wip_nav.dwarf);1831 var buffer: [std.atomic.cache_line]u8 = undefined;
1830 try loc.write(&counter);1832 var counter_bw = std.io.Writer.null.buffered(&buffer);
1833 var counter: ExprLocCounter = .init(wip_nav.dwarf, &counter_bw);
1834 counter.count += try loc.write(&counter);
18311835
1832 const adapter: struct {1836 const adapter: struct {
1833 wip_nav: *WipNav,1837 wip_nav: *WipNav,
...@@ -1844,8 +1848,8 @@ pub const WipNav = struct {...@@ -1844,8 +1848,8 @@ pub const WipNav = struct {
1844 try ctx.wip_nav.sectionOffset(.debug_frame, .debug_info, unit, entry, 0);1848 try ctx.wip_nav.sectionOffset(.debug_frame, .debug_info, unit, entry, 0);
1845 }1849 }
1846 } = .{ .wip_nav = wip_nav };1850 } = .{ .wip_nav = wip_nav };
1847 try uleb128(adapter.writer(), counter.stream.bytes_written);1851 try uleb128(adapter.writer(), counter.count);
1848 try loc.write(adapter);1852 _ = try loc.write(adapter);
1849 }1853 }
18501854
1851 fn frameAddrSym(wip_nav: *WipNav, sym_index: u32, sym_off: u64) UpdateError!void {1855 fn frameAddrSym(wip_nav: *WipNav, sym_index: u32, sym_off: u64) UpdateError!void {
...@@ -6015,15 +6019,21 @@ fn sectionOffsetBytes(dwarf: *Dwarf) u32 {...@@ -6015,15 +6019,21 @@ fn sectionOffsetBytes(dwarf: *Dwarf) u32 {
6015}6019}
60166020
6017fn uleb128Bytes(value: anytype) u32 {6021fn uleb128Bytes(value: anytype) u32 {
6018 var cw = std.io.countingWriter(std.io.null_writer);6022 var buffer: [std.atomic.cache_line]u8 = undefined;
6019 try uleb128(cw.writer(), value);6023 var bw: std.io.BufferedWriter = .{
6020 return @intCast(cw.bytes_written);6024 .unbuffered_writer = .null,
6025 .buffer = .initBuffer(&buffer),
6026 };
6027 return try std.leb.writeUleb128Count(&bw, value);
6021}6028}
60226029
6023fn sleb128Bytes(value: anytype) u32 {6030fn sleb128Bytes(value: anytype) u32 {
6024 var cw = std.io.countingWriter(std.io.null_writer);6031 var buffer: [std.atomic.cache_line]u8 = undefined;
6025 try sleb128(cw.writer(), value);6032 var bw: std.io.BufferedWriter = .{
6026 return @intCast(cw.bytes_written);6033 .unbuffered_writer = .null,
6034 .buffer = .initBuffer(&buffer),
6035 };
6036 return try std.leb.writeIleb128Count(&bw, value);
6027}6037}
60286038
6029/// overrides `-fno-incremental` for testing incremental debug info until `-fincremental` is functional6039/// overrides `-fno-incremental` for testing incremental debug info until `-fincremental` is functional