authorgravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-31 00:39:44+11:00
committergravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-31 02:26:06+11:00
log04a2a4a7cb15f821c52f52efe6b0479f657a704c
tree4078d1486cc843fddf36729493b6024a18956d91
parent0def92cff4650f4c06c338182c20558c855b6234
signature Commit is signed but in an unrecognized format.

std: track decoded string length in std.json tokenizer

1 files changed, 147 insertions(+), 38 deletions(-)

lib/std/json.zig+147-38
......@@ -4,12 +4,21 @@
44
55const std = @import("std.zig");
66const debug = std.debug;
7const assert = debug.assert;
78const testing = std.testing;
89const mem = std.mem;
910const maxInt = std.math.maxInt;
1011
1112pub const WriteStream = @import("json/write_stream.zig").WriteStream;
1213
14const StringEscapes = union(enum) {
15 None,
16
17 Some: struct {
18 size_diff: isize,
19 },
20};
21
1322/// A single token slice into the parent string.
1423///
1524/// Use `token.slice()` on the input at the current position to get the current slice.
......@@ -23,7 +32,14 @@ pub const Token = union(enum) {
2332 count: usize,
2433
2534 /// Whether string contains an escape sequence and cannot be zero-copied
26 has_escape: bool,
35 escapes: StringEscapes,
36
37 pub fn decodedLength(self: @This()) usize {
38 return self.count +% switch (self.escapes) {
39 .None => 0,
40 .Some => |s| @bitCast(usize, s.size_diff),
41 };
42 }
2743
2844 /// Slice into the underlying input string.
2945 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
......@@ -66,7 +82,12 @@ pub const StreamingParser = struct {
6682 // If we stopped now, would the complete parsed string to now be a valid json string
6783 complete: bool,
6884 // Current token flags to pass through to the next generated, see Token.
69 string_has_escape: bool,
85 string_escapes: StringEscapes,
86 // When in .String states, was the previous character a high surrogate?
87 string_last_was_high_surrogate: bool,
88 // Used inside of StringEscapeHexUnicode* states
89 string_unicode_codepoint: u21,
90 // When in .Number states, is the number a (still) valid integer?
7091 number_is_integer: bool,
7192
7293 // Bit-stack for nested object/map literals (max 255 nestings).
......@@ -92,8 +113,10 @@ pub const StreamingParser = struct {
92113 p.stack = 0;
93114 p.stack_used = 0;
94115 p.complete = false;
95 p.string_has_escape = false;
96 p.number_is_integer = true;
116 p.string_escapes = undefined;
117 p.string_last_was_high_surrogate = undefined;
118 p.string_unicode_codepoint = undefined;
119 p.number_is_integer = undefined;
97120 }
98121
99122 pub const State = enum {
......@@ -231,7 +254,8 @@ pub const StreamingParser = struct {
231254 p.after_value_state = .TopLevelEnd;
232255 // We don't actually need the following since after_value_state should override.
233256 p.after_string_state = .ValueEnd;
234 p.string_has_escape = false;
257 p.string_escapes = .None;
258 p.string_last_was_high_surrogate = false;
235259 p.count = 0;
236260 },
237261 't' => {
......@@ -367,6 +391,8 @@ pub const StreamingParser = struct {
367391 },
368392 '"' => {
369393 p.state = .String;
394 p.string_escapes = .None;
395 p.string_last_was_high_surrogate = false;
370396 p.count = 0;
371397 },
372398 't' => {
......@@ -436,6 +462,8 @@ pub const StreamingParser = struct {
436462 },
437463 '"' => {
438464 p.state = .String;
465 p.string_escapes = .None;
466 p.string_last_was_high_surrogate = false;
439467 p.count = 0;
440468 },
441469 't' => {
......@@ -534,15 +562,24 @@ pub const StreamingParser = struct {
534562 token.* = .{
535563 .String = .{
536564 .count = p.count - 1,
537 .has_escape = p.string_has_escape,
565 .escapes = p.string_escapes,
538566 },
539567 };
568 p.string_escapes = undefined;
569 p.string_last_was_high_surrogate = undefined;
540570 },
541571 '\\' => {
542572 p.state = .StringEscapeCharacter;
573 switch (p.string_escapes) {
574 .None => {
575 p.string_escapes = .{ .Some = .{ .size_diff = 0 } };
576 },
577 .Some => {},
578 }
543579 },
544580 0x20, 0x21, 0x23...0x5B, 0x5D...0x7F => {
545581 // non-control ascii
582 p.string_last_was_high_surrogate = false;
546583 },
547584 0xC0...0xDF => {
548585 p.state = .StringUtf8Byte1;
......@@ -569,7 +606,10 @@ pub const StreamingParser = struct {
569606 },
570607
571608 .StringUtf8Byte1 => switch (c >> 6) {
572 0b10 => p.state = .String,
609 0b10 => {
610 p.state = .String;
611 p.string_last_was_high_surrogate = false;
612 },
573613 else => return error.InvalidUtf8Byte,
574614 },
575615
......@@ -583,11 +623,11 @@ pub const StreamingParser = struct {
583623 // however, so we default to the status quo where both are accepted until this
584624 // is further clarified.
585625 '"', '\\', '/', 'b', 'f', 'n', 'r', 't' => {
586 p.string_has_escape = true;
626 p.string_escapes.Some.size_diff -= 1;
587627 p.state = .String;
628 p.string_last_was_high_surrogate = false;
588629 },
589630 'u' => {
590 p.string_has_escape = true;
591631 p.state = .StringEscapeHexUnicode4;
592632 },
593633 else => {
......@@ -595,32 +635,99 @@ pub const StreamingParser = struct {
595635 },
596636 },
597637
598 .StringEscapeHexUnicode4 => switch (c) {
599 '0'...'9', 'A'...'F', 'a'...'f' => {
600 p.state = .StringEscapeHexUnicode3;
601 },
602 else => return error.InvalidUnicodeHexSymbol,
638 .StringEscapeHexUnicode4 => {
639 var codepoint: u21 = undefined;
640 switch (c) {
641 else => return error.InvalidUnicodeHexSymbol,
642 '0'...'9' => {
643 codepoint = c - '0';
644 },
645 'A'...'F' => {
646 codepoint = c - 'A' + 10;
647 },
648 'a'...'f' => {
649 codepoint = c - 'a' + 10;
650 },
651 }
652 p.state = .StringEscapeHexUnicode3;
653 p.string_unicode_codepoint = codepoint << 12;
603654 },
604655
605 .StringEscapeHexUnicode3 => switch (c) {
606 '0'...'9', 'A'...'F', 'a'...'f' => {
607 p.state = .StringEscapeHexUnicode2;
608 },
609 else => return error.InvalidUnicodeHexSymbol,
656 .StringEscapeHexUnicode3 => {
657 var codepoint: u21 = undefined;
658 switch (c) {
659 else => return error.InvalidUnicodeHexSymbol,
660 '0'...'9' => {
661 codepoint = c - '0';
662 },
663 'A'...'F' => {
664 codepoint = c - 'A' + 10;
665 },
666 'a'...'f' => {
667 codepoint = c - 'a' + 10;
668 },
669 }
670 p.state = .StringEscapeHexUnicode2;
671 p.string_unicode_codepoint |= codepoint << 8;
610672 },
611673
612 .StringEscapeHexUnicode2 => switch (c) {
613 '0'...'9', 'A'...'F', 'a'...'f' => {
614 p.state = .StringEscapeHexUnicode1;
615 },
616 else => return error.InvalidUnicodeHexSymbol,
674 .StringEscapeHexUnicode2 => {
675 var codepoint: u21 = undefined;
676 switch (c) {
677 else => return error.InvalidUnicodeHexSymbol,
678 '0'...'9' => {
679 codepoint = c - '0';
680 },
681 'A'...'F' => {
682 codepoint = c - 'A' + 10;
683 },
684 'a'...'f' => {
685 codepoint = c - 'a' + 10;
686 },
687 }
688 p.state = .StringEscapeHexUnicode1;
689 p.string_unicode_codepoint |= codepoint << 4;
617690 },
618691
619 .StringEscapeHexUnicode1 => switch (c) {
620 '0'...'9', 'A'...'F', 'a'...'f' => {
621 p.state = .String;
622 },
623 else => return error.InvalidUnicodeHexSymbol,
692 .StringEscapeHexUnicode1 => {
693 var codepoint: u21 = undefined;
694 switch (c) {
695 else => return error.InvalidUnicodeHexSymbol,
696 '0'...'9' => {
697 codepoint = c - '0';
698 },
699 'A'...'F' => {
700 codepoint = c - 'A' + 10;
701 },
702 'a'...'f' => {
703 codepoint = c - 'a' + 10;
704 },
705 }
706 p.state = .String;
707 p.string_unicode_codepoint |= codepoint;
708 if (p.string_unicode_codepoint < 0xD800 or p.string_unicode_codepoint >= 0xE000) {
709 // not part of surrogate pair
710 p.string_escapes.Some.size_diff -= @as(isize, 6 - (std.unicode.utf8CodepointSequenceLength(p.string_unicode_codepoint) catch unreachable));
711 p.string_last_was_high_surrogate = false;
712 } else if (p.string_unicode_codepoint < 0xDC00) {
713 // 'high' surrogate
714 // takes 3 bytes to encode a half surrogate pair into wtf8
715 p.string_escapes.Some.size_diff -= 6 - 3;
716 p.string_last_was_high_surrogate = true;
717 } else {
718 // 'low' surrogate
719 p.string_escapes.Some.size_diff -= 6;
720 if (p.string_last_was_high_surrogate) {
721 // takes 4 bytes to encode a full surrogate pair into utf8
722 // 3 bytes are already reserved by high surrogate
723 p.string_escapes.Some.size_diff -= -1;
724 } else {
725 // takes 3 bytes to encode a half surrogate pair into wtf8
726 p.string_escapes.Some.size_diff -= -3;
727 }
728 p.string_last_was_high_surrogate = false;
729 }
730 p.string_unicode_codepoint = undefined;
624731 },
625732
626733 .Number => {
......@@ -657,6 +764,7 @@ pub const StreamingParser = struct {
657764 .is_integer = p.number_is_integer,
658765 },
659766 };
767 p.number_is_integer = undefined;
660768 return true;
661769 },
662770 }
......@@ -1271,7 +1379,10 @@ pub const Parser = struct {
12711379 // TODO: We don't strictly have to copy values which do not contain any escape
12721380 // characters if flagged with the option.
12731381 const slice = s.slice(input, i);
1274 return Value{ .String = try unescapeStringAlloc(allocator, slice) };
1382 return switch (s.escapes) {
1383 .None => Value{ .String = try mem.dupe(allocator, u8, slice) },
1384 .Some => |some_escapes| Value{ .String = try unescapeStringAlloc(allocator, some_escapes, slice) },
1385 };
12751386 }
12761387
12771388 fn parseNumber(p: *Parser, n: std.meta.TagPayloadType(Token, Token.Number), input: []const u8, i: usize) !Value {
......@@ -1285,13 +1396,10 @@ pub const Parser = struct {
12851396// Unescape a JSON string
12861397// Only to be used on strings already validated by the parser
12871398// (note the unreachable statements and lack of bounds checking)
1288// Optimized for arena allocators, uses Allocator.shrink
1289//
1290// Idea: count how many bytes we will need to allocate in the streaming parser and store it
1291// in the token to avoid allocating too much memory or iterating through the string again
1292// Downside: need to find how many bytes a unicode escape sequence will produce twice
1293fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1294 const output = try alloc.alloc(u8, input.len);
1399fn unescapeStringAlloc(alloc: *Allocator, escapes: std.meta.TagPayloadType(StringEscapes, StringEscapes.Some), input: []const u8) ![]u8 {
1400 const result_size = input.len +% @bitCast(usize, escapes.size_diff);
1401
1402 const output = try alloc.alloc(u8, result_size);
12951403 errdefer alloc.free(output);
12961404
12971405 var inIndex: usize = 0;
......@@ -1347,8 +1455,9 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
13471455 }
13481456 }
13491457 }
1458 assert(outIndex == result_size);
13501459
1351 return alloc.shrink(output, outIndex);
1460 return output;
13521461}
13531462
13541463test "json.parser.dynamic" {