authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2021-10-11 17:17:53+13:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-11-16 16:33:56-05:00
logdcd88ae568a1e9c0315b39801c9ca124e0e9aefc
treee9abd9ed1985287aed82d880fa3e3e1744958118
parentc587be78d7509e81a56c122eb02d19a8d2c5174e

std/json: use bit-stack for nesting instead of large LLVM integer type

The stack has been adjusted so that instead of pushing to index 0 in the integer we push to the current end/index of the underlying integer. This means we don't require a shift for every limb after each push/pop and instead only require a mask/or and add/sub on a single element of the array. Fixes #5959.

1 files changed, 122 insertions(+), 94 deletions(-)

lib/std/json.zig+122-94
...@@ -132,6 +132,69 @@ pub const Token = union(enum) {...@@ -132,6 +132,69 @@ pub const Token = union(enum) {
132 Null,132 Null,
133};133};
134134
135const AggregateContainerType = enum(u1) { object, array };
136
137// A LIFO bit-stack. Tracks which container-types have been entered during parse.
138fn AggregateContainerStack(comptime n: usize) type {
139 return struct {
140 const Self = @This();
141 const TypeInfo = std.builtin.TypeInfo;
142
143 const element_bitcount = 8 * @sizeOf(usize);
144 const element_count = n / element_bitcount;
145 const ElementType = @Type(TypeInfo{ .Int = TypeInfo.Int{ .signedness = .unsigned, .bits = element_bitcount } });
146 const ElementShiftAmountType = std.math.Log2Int(ElementType);
147
148 comptime {
149 std.debug.assert(n % element_bitcount == 0);
150 }
151
152 memory: [element_count]ElementType,
153 len: usize,
154
155 pub fn init(self: *Self) void {
156 self.memory = [_]ElementType{0} ** element_count;
157 self.len = 0;
158 }
159
160 pub fn push(self: *Self, ty: AggregateContainerType) ?void {
161 if (self.len >= n) {
162 return null;
163 }
164
165 const index = self.len / element_bitcount;
166 const sub_index = @intCast(ElementShiftAmountType, self.len % element_bitcount);
167 const clear_mask = ~(@as(ElementType, 1) << sub_index);
168 const set_bits = @as(ElementType, @enumToInt(ty)) << sub_index;
169
170 self.memory[index] &= clear_mask;
171 self.memory[index] |= set_bits;
172 self.len += 1;
173 }
174
175 pub fn peek(self: *Self) ?AggregateContainerType {
176 if (self.len == 0) {
177 return null;
178 }
179
180 const bit_to_extract = self.len - 1;
181 const index = bit_to_extract / element_bitcount;
182 const sub_index = @intCast(ElementShiftAmountType, bit_to_extract % element_bitcount);
183 const bit = @intCast(u1, (self.memory[index] >> sub_index) & 1);
184 return @intToEnum(AggregateContainerType, bit);
185 }
186
187 pub fn pop(self: *Self) ?AggregateContainerType {
188 if (self.peek()) |ty| {
189 self.len -= 1;
190 return ty;
191 }
192
193 return null;
194 }
195 };
196}
197
135/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as198/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
136/// they are encountered. No copies or allocations are performed during parsing and the entire199/// they are encountered. No copies or allocations are performed during parsing and the entire
137/// parsing state requires ~40-50 bytes of stack space.200/// parsing state requires ~40-50 bytes of stack space.
...@@ -140,6 +203,8 @@ pub const Token = union(enum) {...@@ -140,6 +203,8 @@ pub const Token = union(enum) {
140///203///
141/// For a non-byte based wrapper, consider using TokenStream instead.204/// For a non-byte based wrapper, consider using TokenStream instead.
142pub const StreamingParser = struct {205pub const StreamingParser = struct {
206 const default_max_nestings = 256;
207
143 // Current state208 // Current state
144 state: State,209 state: State,
145 // How many bytes we have counted for the current token210 // How many bytes we have counted for the current token
...@@ -160,14 +225,8 @@ pub const StreamingParser = struct {...@@ -160,14 +225,8 @@ pub const StreamingParser = struct {
160 sequence_first_byte: u8 = undefined,225 sequence_first_byte: u8 = undefined,
161 // When in .Number states, is the number a (still) valid integer?226 // When in .Number states, is the number a (still) valid integer?
162 number_is_integer: bool,227 number_is_integer: bool,
163228 // Bit-stack for nested object/map literals (max 256 nestings).
164 // Bit-stack for nested object/map literals (max 255 nestings).229 stack: AggregateContainerStack(default_max_nestings),
165 stack: u256,
166 stack_used: u8,
167
168 const object_bit = 0;
169 const array_bit = 1;
170 const max_stack_size = maxInt(u8);
171230
172 pub fn init() StreamingParser {231 pub fn init() StreamingParser {
173 var p: StreamingParser = undefined;232 var p: StreamingParser = undefined;
...@@ -181,8 +240,7 @@ pub const StreamingParser = struct {...@@ -181,8 +240,7 @@ pub const StreamingParser = struct {
181 // Set before ever read in main transition function240 // Set before ever read in main transition function
182 p.after_string_state = undefined;241 p.after_string_state = undefined;
183 p.after_value_state = .ValueEnd; // handle end of values normally242 p.after_value_state = .ValueEnd; // handle end of values normally
184 p.stack = 0;243 p.stack.init();
185 p.stack_used = 0;
186 p.complete = false;244 p.complete = false;
187 p.string_escapes = undefined;245 p.string_escapes = undefined;
188 p.string_last_was_high_surrogate = undefined;246 p.string_last_was_high_surrogate = undefined;
...@@ -238,11 +296,15 @@ pub const StreamingParser = struct {...@@ -238,11 +296,15 @@ pub const StreamingParser = struct {
238 NullLiteral2,296 NullLiteral2,
239 NullLiteral3,297 NullLiteral3,
240298
241 // Only call this function to generate array/object final state.299 // Given an aggregate container type, return the state which should be entered after
242 pub fn fromInt(x: anytype) State {300 // processing a complete value type.
243 debug.assert(x == 0 or x == 1);301 pub fn fromAggregateContainerType(ty: AggregateContainerType) State {
244 const T = std.meta.Tag(State);302 comptime {
245 return @intToEnum(State, @intCast(T, x));303 std.debug.assert(@enumToInt(AggregateContainerType.object) == @enumToInt(State.ObjectSeparator));
304 std.debug.assert(@enumToInt(AggregateContainerType.array) == @enumToInt(State.ValueEnd));
305 }
306
307 return @intToEnum(State, @enumToInt(ty));
246 }308 }
247 };309 };
248310
...@@ -286,20 +348,14 @@ pub const StreamingParser = struct {...@@ -286,20 +348,14 @@ pub const StreamingParser = struct {
286 switch (p.state) {348 switch (p.state) {
287 .TopLevelBegin => switch (c) {349 .TopLevelBegin => switch (c) {
288 '{' => {350 '{' => {
289 p.stack <<= 1;351 p.stack.push(.object) orelse return error.TooManyNestedItems;
290 p.stack |= object_bit;
291 p.stack_used += 1;
292
293 p.state = .ValueBegin;352 p.state = .ValueBegin;
294 p.after_string_state = .ObjectSeparator;353 p.after_string_state = .ObjectSeparator;
295354
296 token.* = Token.ObjectBegin;355 token.* = Token.ObjectBegin;
297 },356 },
298 '[' => {357 '[' => {
299 p.stack <<= 1;358 p.stack.push(.array) orelse return error.TooManyNestedItems;
300 p.stack |= array_bit;
301 p.stack_used += 1;
302
303 p.state = .ValueBegin;359 p.state = .ValueBegin;
304 p.after_string_state = .ValueEnd;360 p.after_string_state = .ValueEnd;
305361
...@@ -368,21 +424,17 @@ pub const StreamingParser = struct {...@@ -368,21 +424,17 @@ pub const StreamingParser = struct {
368 // NOTE: These are shared in ValueEnd as well, think we can reorder states to424 // NOTE: These are shared in ValueEnd as well, think we can reorder states to
369 // be a bit clearer and avoid this duplication.425 // be a bit clearer and avoid this duplication.
370 '}' => {426 '}' => {
371 // unlikely427 const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
372 if (p.stack & 1 != object_bit) {428
429 if (last_type != .object) {
373 return error.UnexpectedClosingBrace;430 return error.UnexpectedClosingBrace;
374 }431 }
375 if (p.stack_used == 0) {
376 return error.TooManyClosingItems;
377 }
378432
433 _ = p.stack.pop();
379 p.state = .ValueBegin;434 p.state = .ValueBegin;
380 p.after_string_state = State.fromInt(p.stack & 1);435 p.after_string_state = State.fromAggregateContainerType(last_type);
381
382 p.stack >>= 1;
383 p.stack_used -= 1;
384436
385 switch (p.stack_used) {437 switch (p.stack.len) {
386 0 => {438 0 => {
387 p.complete = true;439 p.complete = true;
388 p.state = .TopLevelEnd;440 p.state = .TopLevelEnd;
...@@ -395,20 +447,17 @@ pub const StreamingParser = struct {...@@ -395,20 +447,17 @@ pub const StreamingParser = struct {
395 token.* = Token.ObjectEnd;447 token.* = Token.ObjectEnd;
396 },448 },
397 ']' => {449 ']' => {
398 if (p.stack & 1 != array_bit) {450 const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
451
452 if (last_type != .array) {
399 return error.UnexpectedClosingBracket;453 return error.UnexpectedClosingBracket;
400 }454 }
401 if (p.stack_used == 0) {
402 return error.TooManyClosingItems;
403 }
404455
456 _ = p.stack.pop();
405 p.state = .ValueBegin;457 p.state = .ValueBegin;
406 p.after_string_state = State.fromInt(p.stack & 1);458 p.after_string_state = State.fromAggregateContainerType(last_type);
407459
408 p.stack >>= 1;460 switch (p.stack.len) {
409 p.stack_used -= 1;
410
411 switch (p.stack_used) {
412 0 => {461 0 => {
413 p.complete = true;462 p.complete = true;
414 p.state = .TopLevelEnd;463 p.state = .TopLevelEnd;
...@@ -421,13 +470,7 @@ pub const StreamingParser = struct {...@@ -421,13 +470,7 @@ pub const StreamingParser = struct {
421 token.* = Token.ArrayEnd;470 token.* = Token.ArrayEnd;
422 },471 },
423 '{' => {472 '{' => {
424 if (p.stack_used == max_stack_size) {473 p.stack.push(.object) orelse return error.TooManyNestedItems;
425 return error.TooManyNestedItems;
426 }
427
428 p.stack <<= 1;
429 p.stack |= object_bit;
430 p.stack_used += 1;
431474
432 p.state = .ValueBegin;475 p.state = .ValueBegin;
433 p.after_string_state = .ObjectSeparator;476 p.after_string_state = .ObjectSeparator;
...@@ -435,13 +478,7 @@ pub const StreamingParser = struct {...@@ -435,13 +478,7 @@ pub const StreamingParser = struct {
435 token.* = Token.ObjectBegin;478 token.* = Token.ObjectBegin;
436 },479 },
437 '[' => {480 '[' => {
438 if (p.stack_used == max_stack_size) {481 p.stack.push(.array) orelse return error.TooManyNestedItems;
439 return error.TooManyNestedItems;
440 }
441
442 p.stack <<= 1;
443 p.stack |= array_bit;
444 p.stack_used += 1;
445482
446 p.state = .ValueBegin;483 p.state = .ValueBegin;
447 p.after_string_state = .ValueEnd;484 p.after_string_state = .ValueEnd;
...@@ -492,13 +529,7 @@ pub const StreamingParser = struct {...@@ -492,13 +529,7 @@ pub const StreamingParser = struct {
492 // TODO: A bit of duplication here and in the following state, redo.529 // TODO: A bit of duplication here and in the following state, redo.
493 .ValueBeginNoClosing => switch (c) {530 .ValueBeginNoClosing => switch (c) {
494 '{' => {531 '{' => {
495 if (p.stack_used == max_stack_size) {532 p.stack.push(.object) orelse return error.TooManyNestedItems;
496 return error.TooManyNestedItems;
497 }
498
499 p.stack <<= 1;
500 p.stack |= object_bit;
501 p.stack_used += 1;
502533
503 p.state = .ValueBegin;534 p.state = .ValueBegin;
504 p.after_string_state = .ObjectSeparator;535 p.after_string_state = .ObjectSeparator;
...@@ -506,13 +537,7 @@ pub const StreamingParser = struct {...@@ -506,13 +537,7 @@ pub const StreamingParser = struct {
506 token.* = Token.ObjectBegin;537 token.* = Token.ObjectBegin;
507 },538 },
508 '[' => {539 '[' => {
509 if (p.stack_used == max_stack_size) {540 p.stack.push(.array) orelse return error.TooManyNestedItems;
510 return error.TooManyNestedItems;
511 }
512
513 p.stack <<= 1;
514 p.stack |= array_bit;
515 p.stack_used += 1;
516541
517 p.state = .ValueBegin;542 p.state = .ValueBegin;
518 p.after_string_state = .ValueEnd;543 p.after_string_state = .ValueEnd;
...@@ -562,24 +587,22 @@ pub const StreamingParser = struct {...@@ -562,24 +587,22 @@ pub const StreamingParser = struct {
562587
563 .ValueEnd => switch (c) {588 .ValueEnd => switch (c) {
564 ',' => {589 ',' => {
565 p.after_string_state = State.fromInt(p.stack & 1);590 const last_type = p.stack.peek() orelse unreachable;
591 p.after_string_state = State.fromAggregateContainerType(last_type);
566 p.state = .ValueBeginNoClosing;592 p.state = .ValueBeginNoClosing;
567 },593 },
568 ']' => {594 ']' => {
569 if (p.stack & 1 != array_bit) {595 const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
596
597 if (last_type != .array) {
570 return error.UnexpectedClosingBracket;598 return error.UnexpectedClosingBracket;
571 }599 }
572 if (p.stack_used == 0) {
573 return error.TooManyClosingItems;
574 }
575600
601 _ = p.stack.pop();
576 p.state = .ValueEnd;602 p.state = .ValueEnd;
577 p.after_string_state = State.fromInt(p.stack & 1);603 p.after_string_state = State.fromAggregateContainerType(last_type);
578
579 p.stack >>= 1;
580 p.stack_used -= 1;
581604
582 if (p.stack_used == 0) {605 if (p.stack.len == 0) {
583 p.complete = true;606 p.complete = true;
584 p.state = .TopLevelEnd;607 p.state = .TopLevelEnd;
585 }608 }
...@@ -587,21 +610,17 @@ pub const StreamingParser = struct {...@@ -587,21 +610,17 @@ pub const StreamingParser = struct {
587 token.* = Token.ArrayEnd;610 token.* = Token.ArrayEnd;
588 },611 },
589 '}' => {612 '}' => {
590 // unlikely613 const last_type = p.stack.peek() orelse return error.TooManyClosingItems;
591 if (p.stack & 1 != object_bit) {614
615 if (last_type != .object) {
592 return error.UnexpectedClosingBrace;616 return error.UnexpectedClosingBrace;
593 }617 }
594 if (p.stack_used == 0) {
595 return error.TooManyClosingItems;
596 }
597618
619 _ = p.stack.pop();
598 p.state = .ValueEnd;620 p.state = .ValueEnd;
599 p.after_string_state = State.fromInt(p.stack & 1);621 p.after_string_state = State.fromAggregateContainerType(last_type);
600622
601 p.stack >>= 1;623 if (p.stack.len == 0) {
602 p.stack_used -= 1;
603
604 if (p.stack_used == 0) {
605 p.complete = true;624 p.complete = true;
606 p.state = .TopLevelEnd;625 p.state = .TopLevelEnd;
607 }626 }
...@@ -1082,6 +1101,15 @@ pub const StreamingParser = struct {...@@ -1082,6 +1101,15 @@ pub const StreamingParser = struct {
1082 }1101 }
1083};1102};
10841103
1104test "json.serialize issue #5959" {
1105 var parser: StreamingParser = undefined;
1106 // StreamingParser has multiple internal fields set to undefined. This causes issues when using
1107 // expectEqual so these are zeroed. We are testing for equality here only because this is a
1108 // known small test reproduction which hits the relevant LLVM issue.
1109 std.mem.set(u8, @ptrCast([*]u8, &parser)[0..@sizeOf(StreamingParser)], 0);
1110 try std.testing.expectEqual(parser, parser);
1111}
1112
1085/// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.1113/// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.
1086pub const TokenStream = struct {1114pub const TokenStream = struct {
1087 i: usize,1115 i: usize,
...@@ -1100,8 +1128,8 @@ pub const TokenStream = struct {...@@ -1100,8 +1128,8 @@ pub const TokenStream = struct {
1100 };1128 };
1101 }1129 }
11021130
1103 fn stackUsed(self: *TokenStream) u8 {1131 fn stackUsed(self: *TokenStream) usize {
1104 return self.parser.stack_used + if (self.token != null) @as(u8, 1) else 0;1132 return self.parser.stack.len + if (self.token != null) @as(usize, 1) else 0;
1105 }1133 }
11061134
1107 pub fn next(self: *TokenStream) Error!?Token {1135 pub fn next(self: *TokenStream) Error!?Token {
...@@ -1490,7 +1518,7 @@ test "skipValue" {...@@ -1490,7 +1518,7 @@ test "skipValue" {
1490 try skipValue(&TokenStream.init("{\"foo\": \"bar\"}"));1518 try skipValue(&TokenStream.init("{\"foo\": \"bar\"}"));
14911519
1492 { // An absurd number of nestings1520 { // An absurd number of nestings
1493 const nestings = 256;1521 const nestings = StreamingParser.default_max_nestings + 1;
14941522
1495 try testing.expectError(1523 try testing.expectError(
1496 error.TooManyNestedItems,1524 error.TooManyNestedItems,
...@@ -1499,7 +1527,7 @@ test "skipValue" {...@@ -1499,7 +1527,7 @@ test "skipValue" {
1499 }1527 }
15001528
1501 { // Would a number token cause problems in a deeply-nested array?1529 { // Would a number token cause problems in a deeply-nested array?
1502 const nestings = 255;1530 const nestings = StreamingParser.default_max_nestings;
1503 const deeply_nested_array = "[" ** nestings ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ "]" ** nestings;1531 const deeply_nested_array = "[" ** nestings ++ "0.118, 999, 881.99, 911.9, 725, 3" ++ "]" ** nestings;
15041532
1505 try skipValue(&TokenStream.init(deeply_nested_array));1533 try skipValue(&TokenStream.init(deeply_nested_array));