authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-06 19:19:57-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-09-06 19:19:57-04:00
logd7268cbb242d6bea6b6e4d929a8d6b99608b640a
tree73a055b7bdbad0babf52cabae6a743c9e9a534a2
parenta8a806e925ddb2362e0021bd83dd73f7389a2dbb
parent3c8e1bc25b6d122ce1295e2e33bb9f54ae801ec0
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6232 from LemonBoy/fix-readall

std: Don't trust stat() size in readAllAlloc fns

7 files changed, 63 insertions(+), 23 deletions(-)

lib/std/array_list.zig+10-2
......@@ -46,7 +46,11 @@ pub fn ArrayListAligned(comptime T: type, comptime alignment: ?u29) type {
4646 /// Deinitialize with `deinit` or use `toOwnedSlice`.
4747 pub fn initCapacity(allocator: *Allocator, num: usize) !Self {
4848 var self = Self.init(allocator);
49 try self.ensureCapacity(num);
49
50 const new_memory = try self.allocator.allocAdvanced(T, alignment, num, .at_least);
51 self.items.ptr = new_memory.ptr;
52 self.capacity = new_memory.len;
53
5054 return self;
5155 }
5256
......@@ -366,7 +370,11 @@ pub fn ArrayListAlignedUnmanaged(comptime T: type, comptime alignment: ?u29) typ
366370 /// Deinitialize with `deinit` or use `toOwnedSlice`.
367371 pub fn initCapacity(allocator: *Allocator, num: usize) !Self {
368372 var self = Self{};
369 try self.ensureCapacity(allocator, num);
373
374 const new_memory = try self.allocator.allocAdvanced(T, alignment, num, .at_least);
375 self.items.ptr = new_memory.ptr;
376 self.capacity = new_memory.len;
377
370378 return self;
371379 }
372380
lib/std/fs.zig+9-3
......@@ -1437,26 +1437,32 @@ pub const Dir = struct {
14371437 /// On success, caller owns returned buffer.
14381438 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
14391439 pub fn readFileAlloc(self: Dir, allocator: *mem.Allocator, file_path: []const u8, max_bytes: usize) ![]u8 {
1440 return self.readFileAllocOptions(allocator, file_path, max_bytes, @alignOf(u8), null);
1440 return self.readFileAllocOptions(allocator, file_path, max_bytes, null, @alignOf(u8), null);
14411441 }
14421442
14431443 /// On success, caller owns returned buffer.
14441444 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
1445 /// If `size_hint` is specified the initial buffer size is calculated using
1446 /// that value, otherwise the effective file size is used instead.
14451447 /// Allows specifying alignment and a sentinel value.
14461448 pub fn readFileAllocOptions(
14471449 self: Dir,
14481450 allocator: *mem.Allocator,
14491451 file_path: []const u8,
14501452 max_bytes: usize,
1453 size_hint: ?usize,
14511454 comptime alignment: u29,
14521455 comptime optional_sentinel: ?u8,
14531456 ) !(if (optional_sentinel) |s| [:s]align(alignment) u8 else []align(alignment) u8) {
14541457 var file = try self.openFile(file_path, .{});
14551458 defer file.close();
14561459
1457 const stat_size = try file.getEndPos();
1460 // If the file size doesn't fit a usize it'll be certainly greater than
1461 // `max_bytes`
1462 const stat_size = size_hint orelse math.cast(usize, try file.getEndPos()) catch
1463 return error.FileTooBig;
14581464
1459 return file.readAllAllocOptions(allocator, stat_size, max_bytes, alignment, optional_sentinel);
1465 return file.readToEndAllocOptions(allocator, max_bytes, stat_size, alignment, optional_sentinel);
14601466 }
14611467
14621468 pub const DeleteTreeError = error{
lib/std/fs/file.zig+29-11
......@@ -363,31 +363,49 @@ pub const File = struct {
363363 try os.futimens(self.handle, &times);
364364 }
365365
366 /// Reads all the bytes from the current position to the end of the file.
366367 /// On success, caller owns returned buffer.
367368 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
368 pub fn readAllAlloc(self: File, allocator: *mem.Allocator, stat_size: u64, max_bytes: usize) ![]u8 {
369 return self.readAllAllocOptions(allocator, stat_size, max_bytes, @alignOf(u8), null);
369 pub fn readToEndAlloc(self: File, allocator: *mem.Allocator, max_bytes: usize) ![]u8 {
370 return self.readToEndAllocOptions(allocator, max_bytes, null, @alignOf(u8), null);
370371 }
371372
373 /// Reads all the bytes from the current position to the end of the file.
372374 /// On success, caller owns returned buffer.
373375 /// If the file is larger than `max_bytes`, returns `error.FileTooBig`.
376 /// If `size_hint` is specified the initial buffer size is calculated using
377 /// that value, otherwise an arbitrary value is used instead.
374378 /// Allows specifying alignment and a sentinel value.
375 pub fn readAllAllocOptions(
379 pub fn readToEndAllocOptions(
376380 self: File,
377381 allocator: *mem.Allocator,
378 stat_size: u64,
379382 max_bytes: usize,
383 size_hint: ?usize,
380384 comptime alignment: u29,
381385 comptime optional_sentinel: ?u8,
382386 ) !(if (optional_sentinel) |s| [:s]align(alignment) u8 else []align(alignment) u8) {
383 const size = math.cast(usize, stat_size) catch math.maxInt(usize);
384 if (size > max_bytes) return error.FileTooBig;
385
386 const buf = try allocator.allocWithOptions(u8, size, alignment, optional_sentinel);
387 errdefer allocator.free(buf);
387 // If no size hint is provided fall back to the size=0 code path
388 const size = size_hint orelse 0;
389
390 // The file size returned by stat is used as hint to set the buffer
391 // size. If the reported size is zero, as it happens on Linux for files
392 // in /proc, a small buffer is allocated instead.
393 const initial_cap = (if (size > 0) size else 1024) + @boolToInt(optional_sentinel != null);
394 var array_list = try std.ArrayListAligned(u8, alignment).initCapacity(allocator, initial_cap);
395 defer array_list.deinit();
396
397 self.reader().readAllArrayList(&array_list, max_bytes) catch |err| switch (err) {
398 error.StreamTooLong => return error.FileTooBig,
399 else => |e| return e,
400 };
388401
389 try self.reader().readNoEof(buf);
390 return buf;
402 if (optional_sentinel) |sentinel| {
403 try array_list.append(sentinel);
404 const buf = array_list.toOwnedSlice();
405 return buf[0 .. buf.len - 1 :sentinel];
406 } else {
407 return array_list.toOwnedSlice();
408 }
391409 }
392410
393411 pub const ReadError = os.ReadError;
lib/std/fs/test.zig+5-5
......@@ -188,30 +188,30 @@ test "readAllAlloc" {
188188 var file = try tmp_dir.dir.createFile("test_file", .{ .read = true });
189189 defer file.close();
190190
191 const buf1 = try file.readAllAlloc(testing.allocator, 0, 1024);
191 const buf1 = try file.readToEndAlloc(testing.allocator, 1024);
192192 defer testing.allocator.free(buf1);
193193 testing.expect(buf1.len == 0);
194194
195195 const write_buf: []const u8 = "this is a test.\nthis is a test.\nthis is a test.\nthis is a test.\n";
196196 try file.writeAll(write_buf);
197197 try file.seekTo(0);
198 const file_size = try file.getEndPos();
199198
200199 // max_bytes > file_size
201 const buf2 = try file.readAllAlloc(testing.allocator, file_size, 1024);
200 const buf2 = try file.readToEndAlloc(testing.allocator, 1024);
202201 defer testing.allocator.free(buf2);
203202 testing.expectEqual(write_buf.len, buf2.len);
204203 testing.expect(std.mem.eql(u8, write_buf, buf2));
205204 try file.seekTo(0);
206205
207206 // max_bytes == file_size
208 const buf3 = try file.readAllAlloc(testing.allocator, file_size, write_buf.len);
207 const buf3 = try file.readToEndAlloc(testing.allocator, write_buf.len);
209208 defer testing.allocator.free(buf3);
210209 testing.expectEqual(write_buf.len, buf3.len);
211210 testing.expect(std.mem.eql(u8, write_buf, buf3));
211 try file.seekTo(0);
212212
213213 // max_bytes < file_size
214 testing.expectError(error.FileTooBig, file.readAllAlloc(testing.allocator, file_size, write_buf.len - 1));
214 testing.expectError(error.FileTooBig, file.readToEndAlloc(testing.allocator, write_buf.len - 1));
215215}
216216
217217test "directory operations on files" {
src-self-hosted/Module.zig+2
......@@ -626,6 +626,7 @@ pub const Scope = struct {
626626 module.gpa,
627627 self.sub_file_path,
628628 std.math.maxInt(u32),
629 null,
629630 1,
630631 0,
631632 );
......@@ -723,6 +724,7 @@ pub const Scope = struct {
723724 module.gpa,
724725 self.sub_file_path,
725726 std.math.maxInt(u32),
727 null,
726728 1,
727729 0,
728730 );
src-self-hosted/main.zig+8-1
......@@ -744,6 +744,7 @@ const FmtError = error{
744744 LinkQuotaExceeded,
745745 FileBusy,
746746 EndOfStream,
747 Unseekable,
747748 NotOpenForWriting,
748749} || fs.File.OpenError;
749750
......@@ -807,7 +808,13 @@ fn fmtPathFile(
807808 if (stat.kind == .Directory)
808809 return error.IsDir;
809810
810 const source_code = source_file.readAllAlloc(fmt.gpa, stat.size, max_src_size) catch |err| switch (err) {
811 const source_code = source_file.readToEndAllocOptions(
812 fmt.gpa,
813 max_src_size,
814 stat.size,
815 @alignOf(u8),
816 null,
817 ) catch |err| switch (err) {
811818 error.ConnectionResetByPeer => unreachable,
812819 error.ConnectionTimedOut => unreachable,
813820 error.NotOpenForReading => unreachable,
src-self-hosted/stage2.zig-1
......@@ -615,7 +615,6 @@ export fn stage2_libc_parse(stage1_libc: *Stage2LibCInstallation, libc_file_z: [
615615 error.NotOpenForWriting => unreachable,
616616 error.NotOpenForReading => unreachable,
617617 error.Unexpected => return .Unexpected,
618 error.EndOfStream => return .EndOfFile,
619618 error.IsDir => return .IsDir,
620619 error.ConnectionResetByPeer => unreachable,
621620 error.ConnectionTimedOut => unreachable,