authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2021-10-11 17:17:53+13:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-11-16 16:33:56-05:00
logdcd88ae568a1e9c0315b39801c9ca124e0e9aefc
treee9abd9ed1985287aed82d880fa3e3e1744958118
parentc587be78d7509e81a56c122eb02d19a8d2c5174e

std/json: use bit-stack for nesting instead of large LLVM integer type

The stack has been adjusted so that instead of pushing to index 0 in the integer we push to the current end/index of the underlying integer. This means we don't require a shift for every limb after each push/pop and instead only require a mask/or and add/sub on a single element of the array. Fixes #5959.

1 files changed, 122 insertions(+), 94 deletions(-)

lib/std/json.zig+122-94
......@@ -132,6 +132,69 @@ pub const Token = union(enum) {
132132 Null,
133133};
134134
135const AggregateContainerType = enum(u1) { object, array };
136
137// A LIFO bit-stack. Tracks which container-types have been entered during parse.
138fn AggregateContainerStack(comptime n: usize) type {
139 return struct {
140 const Self = @This();
141 const TypeInfo = std.builtin.TypeInfo;
142
143 const element_bitcount = 8 * @sizeOf(usize);
144 const element_count = n / element_bitcount;
145 const ElementType = @Type(TypeInfo{ .Int = TypeInfo.Int{ .signedness = .unsigned, .bits = element_bitcount } });
146 const ElementShiftAmountType = std.math.Log2Int(ElementType);
147
148 comptime {
149 std.debug.assert(n % element_bitcount == 0);
150 }
151
152 memory: [element_count]ElementType,
153 len: usize,
154
155 pub fn init(self: *Self) void {
156 self.memory = [_]ElementType{0} ** element_count;
157 self.len = 0;
158 }
159
160 pub fn push(self: *Self, ty: AggregateContainerType) ?void {
161 if (self.len >= n) {
162 return null;
163 }
164
165 const index = self.len / element_bitcount;
166 const sub_index = @intCast(ElementShiftAmountType, self.len % element_bitcount);
167 const clear_mask = ~(@as(ElementType, 1) << sub_index);
168 const set_bits = @as(ElementType, @enumToInt(ty)) << sub_index;
169
170 self.memory[index] &= clear_mask;
171 self.memory[index] |= set_bits;
172 self.len += 1;
173 }
174
175 pub fn peek(self: *Self) ?AggregateContainerType {
176 if (self.len == 0) {
177 return null;
178 }
179
180 const bit_to_extract = self.len - 1;
181 const index = bit_to_extract / element_bitcount;
182 const sub_index = @intCast(ElementShiftAmountType, bit_to_extract % element_bitcount);
183 const bit = @intCast(u1, (self.memory[index] >> sub_index) & 1);
184 return @intToEnum(AggregateContainerType, bit);
185 }
186
187 pub fn pop(self: *Self) ?AggregateContainerType {
188 if (self.peek()) |ty| {
189 self.len -= 1;
190 return ty;
191 }
192
193 return null;
194 }
195 };
196}
197
135198/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
136199/// they are encountered. No copies or allocations are performed during parsing and the entire
137200/// parsing state requires ~40-50 bytes of stack space.
......@@ -140,6 +203,8 @@ pub const Token = union(enum) {
140203///
141204/// For a non-byte based wrapper, consider using TokenStream instead.
142205pub const StreamingParser = struct {
206 const default_max_nestings = 256;
207
143208 // Current state
144209 state: State,
145210 // How many bytes we have counted for the current token
......@@ -160,14 +225,8 @@ pub const StreamingParser = struct {
160225 sequence_first_byte: u8 = undefined,
161226 // When in .Number states, is the number a (still) valid integer?
162227 number_is_integer: bool,
163
164 // Bit-stack for nested object/map literals (max 255 nestings).
165 stack: u256,
166 stack_used: u8,
167
168 const object_bit = 0;
169 const array_bit = 1;
170 const max_stack_size = maxInt(u8);
228 // Bit-stack for nested object/map literals (max 256 nestings).
229 stack: AggregateContainerStack(default_max_nestings),
171230
172231 pub fn init() StreamingParser {
173232 var p: StreamingParser = undefined;
......@@ -181,8 +240,7 @@ pub const StreamingParser = struct {
181240 // Set before ever read in main transition function
182241 p.after_string_state = undefined;
183242 p.after_value_state = .ValueEnd; // handle end of values normally
184 p.stack = 0;
185 p.stack_used = 0;
243 p.stack.init();
186244 p.complete = false;
187245 p.string_escapes = undefined;
188246 p.string_last_was_high_surrogate = undefined;
......@@ -238,11 +296,15 @@ pub const StreamingParser = struct {
238296 NullLiteral2,
239297 NullLiteral3,
240298
241 // Only call this function to generate array/object final state.
242 pub fn fromInt(x: anytype) State {
243 debug.assert(x == 0 or x == 1);
244 const T = std.meta.Tag(State);
245 return @intToEnum(State, @intCast(T, x));
299 // Given an aggregate container type, return the state which should be entered after
300 // processing a complete value type.
301 pub fn fromAggregateContainerType(ty: AggregateContainerType) State {
302 comptime {
303 std.debug.assert(@enumToInt(AggregateContainerType.object) == @enumToInt(State.ObjectSeparator));
304 std.debug.assert(@enumToInt(AggregateContainerType.array) == @enumToInt(State.ValueEnd));
305 }
306
307 return @intToEnum(State, @enumToInt(ty));
246308 }
247309 };
248310
......@@ -286,20 +348,14 @@ pub const StreamingParser = struct {
286348 switch (p.state) {
287349 .TopLevelBegin => switch (c) {
288350 '{' => {
289 p.stack <<= 1;
290 p.stack |= object_bit;
291 p.stack_used += 1;
292
351 p.stack.push(.object) orelse return error.TooManyNestedItems;
293352 p.state = .ValueBegin;
294353 p.after_string_state = .ObjectSeparator;
295354
296355 token.* = Token.ObjectBegin;
297356 },
298357 '[' => {
299 p.stack <<= 1;
300 p.stack |= array_bit;
301 p.stack_used += 1;
302
358 p.stack.push(.array) orelse return error.TooManyNestedItems;
303359 p.state = .ValueBegin;
304360 p.after_string_state = .ValueEnd;
305361
......@@ -368,21 +424,17 @@ pub const StreamingParser = struct {
368424 // NOTE: These are shared in ValueEnd as well, think we can reorder states to
369425 // be a bit clearer and avoid this duplication.
370426 '}' => {
371 // unlikely
372 if (p.stack & 1 != object_bit) {
427 const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
428
429 if (last_type != .object) {
373430 return error.UnexpectedClosingBrace;
374431 }
375 if (p.stack_used == 0) {
376 return error.TooManyClosingItems;
377 }
378432
433 _ = p.stack.pop();
379434 p.state = .ValueBegin;
380 p.after_string_state = State.fromInt(p.stack & 1);
381
382 p.stack >>= 1;
383 p.stack_used -= 1;
435 p.after_string_state = State.fromAggregateContainerType(last_type);
384436
385 switch (p.stack_used) {
437 switch (p.stack.len) {
386438 0 => {
387439 p.complete = true;
388440 p.state = .TopLevelEnd;
......@@ -395,20 +447,17 @@ pub const StreamingParser = struct {
395447 token.* = Token.ObjectEnd;
396448 },
397449 ']' => {
398 if (p.stack & 1 != array_bit) {
450 const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
451
452 if (last_type != .array) {
399453 return error.UnexpectedClosingBracket;
400454 }
401 if (p.stack_used == 0) {
402 return error.TooManyClosingItems;
403 }
404455
456 _ = p.stack.pop();
405457 p.state = .ValueBegin;
406 p.after_string_state = State.fromInt(p.stack & 1);
458 p.after_string_state = State.fromAggregateContainerType(last_type);
407459
408 p.stack >>= 1;
409 p.stack_used -= 1;
410
411 switch (p.stack_used) {
460 switch (p.stack.len) {
412461 0 => {
413462 p.complete = true;
414463 p.state = .TopLevelEnd;
......@@ -421,13 +470,7 @@ pub const StreamingParser = struct {
421470 token.* = Token.ArrayEnd;
422471 },
423472 '{' => {
424 if (p.stack_used == max_stack_size) {
425 return error.TooManyNestedItems;
426 }
427
428 p.stack <<= 1;
429 p.stack |= object_bit;
430 p.stack_used += 1;
473 p.stack.push(.object) orelse return error.TooManyNestedItems;
431474
432475 p.state = .ValueBegin;
433476 p.after_string_state = .ObjectSeparator;
......@@ -435,13 +478,7 @@ pub const StreamingParser = struct {
435478 token.* = Token.ObjectBegin;
436479 },
437480 '[' => {
438 if (p.stack_used == max_stack_size) {
439 return error.TooManyNestedItems;
440 }
441
442 p.stack <<= 1;
443 p.stack |= array_bit;
444 p.stack_used += 1;
481 p.stack.push(.array) orelse return error.TooManyNestedItems;
445482
446483 p.state = .ValueBegin;
447484 p.after_string_state = .ValueEnd;
......@@ -492,13 +529,7 @@ pub const StreamingParser = struct {
492529 // TODO: A bit of duplication here and in the following state, redo.
493530 .ValueBeginNoClosing => switch (c) {
494531 '{' => {
495 if (p.stack_used == max_stack_size) {
496 return error.TooManyNestedItems;
497 }
498
499 p.stack <<= 1;
500 p.stack |= object_bit;
501 p.stack_used += 1;
532 p.stack.push(.object) orelse return error.TooManyNestedItems;
502533
503534 p.state = .ValueBegin;
504535 p.after_string_state = .ObjectSeparator;
......@@ -506,13 +537,7 @@ pub const StreamingParser = struct {
506537 token.* = Token.ObjectBegin;
507538 },
508539 '[' => {
509 if (p.stack_used == max_stack_size) {
510 return error.TooManyNestedItems;
511 }
512
513 p.stack <<= 1;
514 p.stack |= array_bit;
515 p.stack_used += 1;
540 p.stack.push(.array) orelse return error.TooManyNestedItems;
516541
517542 p.state = .ValueBegin;
518543 p.after_string_state = .ValueEnd;
......@@ -562,24 +587,22 @@ pub const StreamingParser = struct {
562587
563588 .ValueEnd => switch (c) {
564589 ',' => {
565 p.after_string_state = State.fromInt(p.stack & 1);
590 const last_type = p.stack.peek() orelse unreachable;
591 p.after_string_state = State.fromAggregateContainerType(last_type);
566592 p.state = .ValueBeginNoClosing;
567593 },
568594 ']' => {
569 if (p.stack & 1 != array_bit) {
595 const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
596
597 if (last_type != .array) {
570598 return error.UnexpectedClosingBracket;
571599 }
572 if (p.stack_used == 0) {
573 return error.TooManyClosingItems;
574 }
575600
601 _ = p.stack.pop();
576602 p.state = .ValueEnd;
577 p.after_string_state = State.fromInt(p.stack & 1);
578
579 p.stack >>= 1;
580 p.stack_used -= 1;
603 p.after_string_state = State.fromAggregateContainerType(last_type);
581604
582 if (p.stack_used == 0) {
605 if (p.stack.len == 0) {
583606 p.complete = true;
584607 p.state = .TopLevelEnd;
585608 }
......@@ -587,21 +610,17 @@ pub const StreamingParser = struct {
587610 token.* = Token.ArrayEnd;
588611 },
589612 '}' => {
590 // unlikely
591 if (p.stack & 1 != object_bit) {
613 const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
614
615 if (last_type != .object) {
592616 return error.UnexpectedClosingBrace;
593617 }
594 if (p.stack_used == 0) {
595 return error.TooManyClosingItems;
596 }
597618
619 _ = p.stack.pop();
598620 p.state = .ValueEnd;
599 p.after_string_state = State.fromInt(p.stack & 1);
621 p.after_string_state = State.fromAggregateContainerType(last_type);
600622
601 p.stack >>= 1;
602 p.stack_used -= 1;
603
604 if (p.stack_used == 0) {
623 if (p.stack.len == 0) {
605624 p.complete = true;
606625 p.state = .TopLevelEnd;
607626 }
......@@ -1082,6 +1101,15 @@ pub const StreamingParser = struct {
10821101 }
10831102};
10841103
1104test "json.serialize issue #5959" {
1105 var parser: StreamingParser = undefined;
1106 // StreamingParser has multiple internal fields set to undefined. This causes issues when using
1107 // expectEqual so these are zeroed. We are testing for equality here only because this is a
1108 // known small test reproduction which hits the relevant LLVM issue.
1109 std.mem.set(u8, @ptrCast([*]u8, &parser)[0..@sizeOf(StreamingParser)], 0);
1110 try std.testing.expectEqual(parser, parser);
1111}
1112
10851113/// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.
10861114pub const TokenStream = struct {
10871115 i: usize,
......@@ -1100,8 +1128,8 @@ pub const TokenStream = struct {
11001128 };
11011129 }
11021130
1103 fn stackUsed(self: *TokenStream) u8 {
1104 return self.parser.stack_used + if (self.token != null) @as(u8, 1) else 0;
1131 fn stackUsed(self: *TokenStream) usize {
1132 return self.parser.stack.len + if (self.token != null) @as(usize, 1) else 0;
11051133 }
11061134
11071135 pub fn next(self: *TokenStream) Error!?Token {
......@@ -1490,7 +1518,7 @@ test "skipValue" {
14901518 try skipValue(&TokenStream.init("{\"foo\": \"bar\"}"));
14911519
14921520 { // An absurd number of nestings
1493 const nestings = 256;
1521 const nestings = StreamingParser.default_max_nestings + 1;
14941522
14951523 try testing.expectError(
14961524 error.TooManyNestedItems,
......@@ -1499,7 +1527,7 @@ test "skipValue" {
14991527 }
15001528
15011529 { // Would a number token cause problems in a deeply-nested array?
1502 const nestings = 255;
1530 const nestings = StreamingParser.default_max_nestings;
15031531 const deeply_nested_array = "[" ** nestings ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ "]" ** nestings;
15041532
15051533 try skipValue(&TokenStream.init(deeply_nested_array));