authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-11 21:12:47-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-11 21:12:47-05:00
loged4d94a5d54bc49b3661d602301a5ec926abef61
tree13e830ea515e66e9c1a314281ba3c346d3ed9c35
parentc4e7d05ce37d60141afab997f23df33f5d4a218b

self-hosted: test all out of memory conditions

3 files changed, 121 insertions(+), 21 deletions(-)

src-self-hosted/main.zig+36-15
...@@ -8,6 +8,7 @@ const warn = std.debug.warn;...@@ -8,6 +8,7 @@ const warn = std.debug.warn;
8const Tokenizer = @import("tokenizer.zig").Tokenizer;8const Tokenizer = @import("tokenizer.zig").Tokenizer;
9const Token = @import("tokenizer.zig").Token;9const Token = @import("tokenizer.zig").Token;
10const Parser = @import("parser.zig").Parser;10const Parser = @import("parser.zig").Parser;
11const assert = std.debug.assert;
1112
12pub fn main() -> %void {13pub fn main() -> %void {
13 main2() %% |err| {14 main2() %% |err| {
...@@ -68,28 +69,48 @@ pub fn main2() -> %void {...@@ -68,28 +69,48 @@ pub fn main2() -> %void {
6869
69var fixed_buffer_mem: [100 * 1024]u8 = undefined;70var fixed_buffer_mem: [100 * 1024]u8 = undefined;
7071
71fn testCanonical(source: []const u8) {72fn testParse(source: []const u8, allocator: &mem.Allocator) -> %[]u8 {
72 var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
73 const allocator = &fixed_allocator.allocator;
74
75 var tokenizer = Tokenizer.init(source);73 var tokenizer = Tokenizer.init(source);
76 var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");74 var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
77 defer parser.deinit();75 defer parser.deinit();
7876
79 const root_node = parser.parse() %% unreachable;77 const root_node = %return parser.parse();
80 defer parser.freeAst(root_node);78 defer parser.freeAst(root_node);
8179
82 var buffer = std.Buffer.initSize(allocator, 0) %% unreachable;80 var buffer = %return std.Buffer.initSize(allocator, 0);
83 var buffer_out_stream = io.BufferOutStream.init(&buffer);81 var buffer_out_stream = io.BufferOutStream.init(&buffer);
84 parser.renderSource(&buffer_out_stream.stream, root_node) %% unreachable;82 %return parser.renderSource(&buffer_out_stream.stream, root_node);
8583 return buffer.toOwnedSlice();
86 if (!mem.eql(u8, buffer.toSliceConst(), source)) {84}
87 warn("\n====== expected this output: =========\n");85
88 warn("{}", source);86fn testCanonical(source: []const u8) {
89 warn("\n======== instead found this: =========\n");87 const needed_alloc_count = {
90 warn("{}", buffer.toSliceConst());88 // Try it once with unlimited memory, make sure it works
91 warn("\n======================================\n");89 var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
92 @panic("test failed");90 var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, @maxValue(usize));
91 const result_source = testParse(source, &failing_allocator.allocator) %% @panic("test failed");
92 if (!mem.eql(u8, result_source, source)) {
93 warn("\n====== expected this output: =========\n");
94 warn("{}", source);
95 warn("\n======== instead found this: =========\n");
96 warn("{}", result_source);
97 warn("\n======================================\n");
98 @panic("test failed");
99 }
100 failing_allocator.allocator.free(result_source);
101 failing_allocator.index
102 };
103
104 var fail_index = needed_alloc_count;
105 while (fail_index != 0) {
106 fail_index -= 1;
107 var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
108 var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, fail_index);
109 if (testParse(source, &failing_allocator.allocator)) |_| {
110 @panic("non-deterministic memory usage");
111 } else |err| {
112 assert(err == error.OutOfMemory);
113 }
93 }114 }
94}115}
95116
src-self-hosted/parser.zig+29-6
...@@ -18,6 +18,7 @@ pub const Parser = struct {...@@ -18,6 +18,7 @@ pub const Parser = struct {
18 put_back_tokens: [2]Token,18 put_back_tokens: [2]Token,
19 put_back_count: usize,19 put_back_count: usize,
20 source_file_name: []const u8,20 source_file_name: []const u8,
21 cleanup_root_node: ?&ast.NodeRoot,
2122
22 // This memory contents are used only during a function call. It's used to repurpose memory;23 // This memory contents are used only during a function call. It's used to repurpose memory;
23 // specifically so that freeAst can be guaranteed to succeed.24 // specifically so that freeAst can be guaranteed to succeed.
...@@ -32,10 +33,12 @@ pub const Parser = struct {...@@ -32,10 +33,12 @@ pub const Parser = struct {
32 .put_back_count = 0,33 .put_back_count = 0,
33 .source_file_name = source_file_name,34 .source_file_name = source_file_name,
34 .utility_bytes = []align(utility_bytes_align) u8{},35 .utility_bytes = []align(utility_bytes_align) u8{},
36 .cleanup_root_node = null,
35 };37 };
36 }38 }
3739
38 pub fn deinit(self: &Parser) {40 pub fn deinit(self: &Parser) {
41 assert(self.cleanup_root_node == null);
39 self.allocator.free(self.utility_bytes);42 self.allocator.free(self.utility_bytes);
40 }43 }
4144
...@@ -115,13 +118,29 @@ pub const Parser = struct {...@@ -115,13 +118,29 @@ pub const Parser = struct {
115 }118 }
116119
117 pub fn parse(self: &Parser) -> %&ast.NodeRoot {120 pub fn parse(self: &Parser) -> %&ast.NodeRoot {
121 const result = self.parseInner() %% |err| {
122 if (self.cleanup_root_node) |root_node| {
123 self.freeAst(root_node);
124 }
125 err
126 };
127 self.cleanup_root_node = null;
128 return result;
129 }
130
131 pub fn parseInner(self: &Parser) -> %&ast.NodeRoot {
118 var stack = self.initUtilityArrayList(State);132 var stack = self.initUtilityArrayList(State);
119 defer self.deinitUtilityArrayList(stack);133 defer self.deinitUtilityArrayList(stack);
120134
121 const root_node = %return self.createRoot();135 const root_node = {
122 %defer self.allocator.destroy(root_node);136 const root_node = %return self.createRoot();
123 %return stack.append(State.TopLevel);137 %defer self.allocator.destroy(root_node);
124 %defer self.freeAst(root_node);138 // This stack append has to succeed for freeAst to work
139 %return stack.append(State.TopLevel);
140 root_node
141 };
142 assert(self.cleanup_root_node == null);
143 self.cleanup_root_node = root_node;
125144
126 while (true) {145 while (true) {
127 //{146 //{
...@@ -1063,11 +1082,15 @@ pub const Parser = struct {...@@ -1063,11 +1082,15 @@ pub const Parser = struct {
1063 const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);1082 const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
1064 self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);1083 self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
1065 const typed_slice = ([]T)(self.utility_bytes);1084 const typed_slice = ([]T)(self.utility_bytes);
1066 return ArrayList(T).fromOwnedSlice(self.allocator, typed_slice);1085 return ArrayList(T) {
1086 .allocator = self.allocator,
1087 .items = typed_slice,
1088 .len = 0,
1089 };
1067 }1090 }
10681091
1069 fn deinitUtilityArrayList(self: &Parser, list: var) {1092 fn deinitUtilityArrayList(self: &Parser, list: var) {
1070 self.utility_bytes = ([]align(utility_bytes_align) u8)(list.toOwnedSlice());1093 self.utility_bytes = ([]align(utility_bytes_align) u8)(list.items);
1071 }1094 }
10721095
1073};1096};
std/debug.zig+56
...@@ -968,3 +968,59 @@ fn readILeb128(in_stream: &io.InStream) -> %i64 {...@@ -968,3 +968,59 @@ fn readILeb128(in_stream: &io.InStream) -> %i64 {
968pub const global_allocator = &global_fixed_allocator.allocator;968pub const global_allocator = &global_fixed_allocator.allocator;
969var global_fixed_allocator = mem.FixedBufferAllocator.init(global_allocator_mem[0..]);969var global_fixed_allocator = mem.FixedBufferAllocator.init(global_allocator_mem[0..]);
970var global_allocator_mem: [100 * 1024]u8 = undefined;970var global_allocator_mem: [100 * 1024]u8 = undefined;
971
972/// Allocator that fails after N allocations, useful for making sure out of
973/// memory conditions are handled correctly.
974pub const FailingAllocator = struct {
975 allocator: mem.Allocator,
976 index: usize,
977 fail_index: usize,
978 internal_allocator: &mem.Allocator,
979 allocated_bytes: usize,
980
981 pub fn init(allocator: &mem.Allocator, fail_index: usize) -> FailingAllocator {
982 return FailingAllocator {
983 .internal_allocator = allocator,
984 .fail_index = fail_index,
985 .index = 0,
986 .allocated_bytes = 0,
987 .allocator = mem.Allocator {
988 .allocFn = alloc,
989 .reallocFn = realloc,
990 .freeFn = free,
991 },
992 };
993 }
994
995 fn alloc(allocator: &mem.Allocator, n: usize, alignment: u29) -> %[]u8 {
996 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
997 if (self.index == self.fail_index) {
998 return error.OutOfMemory;
999 }
1000 self.index += 1;
1001 const result = %return self.internal_allocator.allocFn(self.internal_allocator, n, alignment);
1002 self.allocated_bytes += result.len;
1003 return result;
1004 }
1005
1006 fn realloc(allocator: &mem.Allocator, old_mem: []u8, new_size: usize, alignment: u29) -> %[]u8 {
1007 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
1008 if (new_size <= old_mem.len) {
1009 self.allocated_bytes -= old_mem.len - new_size;
1010 return self.internal_allocator.reallocFn(self.internal_allocator, old_mem, new_size, alignment);
1011 }
1012 if (self.index == self.fail_index) {
1013 return error.OutOfMemory;
1014 }
1015 self.index += 1;
1016 const result = %return self.internal_allocator.reallocFn(self.internal_allocator, old_mem, new_size, alignment);
1017 self.allocated_bytes += new_size - old_mem.len;
1018 return result;
1019 }
1020
1021 fn free(allocator: &mem.Allocator, bytes: []u8) {
1022 const self = @fieldParentPtr(FailingAllocator, "allocator", allocator);
1023 self.allocated_bytes -= bytes.len;
1024 return self.internal_allocator.freeFn(self.internal_allocator, bytes);
1025 }
1026};