authorgravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-31 00:39:44+11:00
committergravatar for quae@daurnimator.comdaurnimator <quae@daurnimator.com> 2019-12-31 02:26:06+11:00
log04a2a4a7cb15f821c52f52efe6b0479f657a704c
tree4078d1486cc843fddf36729493b6024a18956d91
parent0def92cff4650f4c06c338182c20558c855b6234
signature Commit is signed but in an unrecognized format.

std: track decoded string length in std.json tokenizer

1 files changed, 147 insertions(+), 38 deletions(-)

lib/std/json.zig+147-38
...@@ -4,12 +4,21 @@...@@ -4,12 +4,21 @@
44
5const std = @import("std.zig");5const std = @import("std.zig");
6const debug = std.debug;6const debug = std.debug;
7const assert = debug.assert;
7const testing = std.testing;8const testing = std.testing;
8const mem = std.mem;9const mem = std.mem;
9const maxInt = std.math.maxInt;10const maxInt = std.math.maxInt;
1011
11pub const WriteStream = @import("json/write_stream.zig").WriteStream;12pub const WriteStream = @import("json/write_stream.zig").WriteStream;
1213
14const StringEscapes = union(enum) {
15 None,
16
17 Some: struct {
18 size_diff: isize,
19 },
20};
21
13/// A single token slice into the parent string.22/// A single token slice into the parent string.
14///23///
15/// Use `token.slice()` on the input at the current position to get the current slice.24/// Use `token.slice()` on the input at the current position to get the current slice.
...@@ -23,7 +32,14 @@ pub const Token = union(enum) {...@@ -23,7 +32,14 @@ pub const Token = union(enum) {
23 count: usize,32 count: usize,
2433
25 /// Whether string contains an escape sequence and cannot be zero-copied34 /// Whether string contains an escape sequence and cannot be zero-copied
26 has_escape: bool,35 escapes: StringEscapes,
36
37 pub fn decodedLength(self: @This()) usize {
38 return self.count +% switch (self.escapes) {
39 .None => 0,
40 .Some => |s| @bitCast(usize, s.size_diff),
41 };
42 }
2743
28 /// Slice into the underlying input string.44 /// Slice into the underlying input string.
29 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {45 pub fn slice(self: @This(), input: []const u8, i: usize) []const u8 {
...@@ -66,7 +82,12 @@ pub const StreamingParser = struct {...@@ -66,7 +82,12 @@ pub const StreamingParser = struct {
66 // If we stopped now, would the complete parsed string to now be a valid json string82 // If we stopped now, would the complete parsed string to now be a valid json string
67 complete: bool,83 complete: bool,
68 // Current token flags to pass through to the next generated, see Token.84 // Current token flags to pass through to the next generated, see Token.
69 string_has_escape: bool,85 string_escapes: StringEscapes,
86 // When in .String states, was the previous character a high surrogate?
87 string_last_was_high_surrogate: bool,
88 // Used inside of StringEscapeHexUnicode* states
89 string_unicode_codepoint: u21,
90 // When in .Number states, is the number a (still) valid integer?
70 number_is_integer: bool,91 number_is_integer: bool,
7192
72 // Bit-stack for nested object/map literals (max 255 nestings).93 // Bit-stack for nested object/map literals (max 255 nestings).
...@@ -92,8 +113,10 @@ pub const StreamingParser = struct {...@@ -92,8 +113,10 @@ pub const StreamingParser = struct {
92 p.stack = 0;113 p.stack = 0;
93 p.stack_used = 0;114 p.stack_used = 0;
94 p.complete = false;115 p.complete = false;
95 p.string_has_escape = false;116 p.string_escapes = undefined;
96 p.number_is_integer = true;117 p.string_last_was_high_surrogate = undefined;
118 p.string_unicode_codepoint = undefined;
119 p.number_is_integer = undefined;
97 }120 }
98121
99 pub const State = enum {122 pub const State = enum {
...@@ -231,7 +254,8 @@ pub const StreamingParser = struct {...@@ -231,7 +254,8 @@ pub const StreamingParser = struct {
231 p.after_value_state = .TopLevelEnd;254 p.after_value_state = .TopLevelEnd;
232 // We don't actually need the following since after_value_state should override.255 // We don't actually need the following since after_value_state should override.
233 p.after_string_state = .ValueEnd;256 p.after_string_state = .ValueEnd;
234 p.string_has_escape = false;257 p.string_escapes = .None;
258 p.string_last_was_high_surrogate = false;
235 p.count = 0;259 p.count = 0;
236 },260 },
237 't' => {261 't' => {
...@@ -367,6 +391,8 @@ pub const StreamingParser = struct {...@@ -367,6 +391,8 @@ pub const StreamingParser = struct {
367 },391 },
368 '"' => {392 '"' => {
369 p.state = .String;393 p.state = .String;
394 p.string_escapes = .None;
395 p.string_last_was_high_surrogate = false;
370 p.count = 0;396 p.count = 0;
371 },397 },
372 't' => {398 't' => {
...@@ -436,6 +462,8 @@ pub const StreamingParser = struct {...@@ -436,6 +462,8 @@ pub const StreamingParser = struct {
436 },462 },
437 '"' => {463 '"' => {
438 p.state = .String;464 p.state = .String;
465 p.string_escapes = .None;
466 p.string_last_was_high_surrogate = false;
439 p.count = 0;467 p.count = 0;
440 },468 },
441 't' => {469 't' => {
...@@ -534,15 +562,24 @@ pub const StreamingParser = struct {...@@ -534,15 +562,24 @@ pub const StreamingParser = struct {
534 token.* = .{562 token.* = .{
535 .String = .{563 .String = .{
536 .count = p.count - 1,564 .count = p.count - 1,
537 .has_escape = p.string_has_escape,565 .escapes = p.string_escapes,
538 },566 },
539 };567 };
568 p.string_escapes = undefined;
569 p.string_last_was_high_surrogate = undefined;
540 },570 },
541 '\\' => {571 '\\' => {
542 p.state = .StringEscapeCharacter;572 p.state = .StringEscapeCharacter;
573 switch (p.string_escapes) {
574 .None => {
575 p.string_escapes = .{ .Some = .{ .size_diff = 0 } };
576 },
577 .Some => {},
578 }
543 },579 },
544 0x20, 0x21, 0x23...0x5B, 0x5D...0x7F => {580 0x20, 0x21, 0x23...0x5B, 0x5D...0x7F => {
545 // non-control ascii581 // non-control ascii
582 p.string_last_was_high_surrogate = false;
546 },583 },
547 0xC0...0xDF => {584 0xC0...0xDF => {
548 p.state = .StringUtf8Byte1;585 p.state = .StringUtf8Byte1;
...@@ -569,7 +606,10 @@ pub const StreamingParser = struct {...@@ -569,7 +606,10 @@ pub const StreamingParser = struct {
569 },606 },
570607
571 .StringUtf8Byte1 => switch (c >> 6) {608 .StringUtf8Byte1 => switch (c >> 6) {
572 0b10 => p.state = .String,609 0b10 => {
610 p.state = .String;
611 p.string_last_was_high_surrogate = false;
612 },
573 else => return error.InvalidUtf8Byte,613 else => return error.InvalidUtf8Byte,
574 },614 },
575615
...@@ -583,11 +623,11 @@ pub const StreamingParser = struct {...@@ -583,11 +623,11 @@ pub const StreamingParser = struct {
583 // however, so we default to the status quo where both are accepted until this623 // however, so we default to the status quo where both are accepted until this
584 // is further clarified.624 // is further clarified.
585 '"', '\\', '/', 'b', 'f', 'n', 'r', 't' => {625 '"', '\\', '/', 'b', 'f', 'n', 'r', 't' => {
586 p.string_has_escape = true;626 p.string_escapes.Some.size_diff -= 1;
587 p.state = .String;627 p.state = .String;
628 p.string_last_was_high_surrogate = false;
588 },629 },
589 'u' => {630 'u' => {
590 p.string_has_escape = true;
591 p.state = .StringEscapeHexUnicode4;631 p.state = .StringEscapeHexUnicode4;
592 },632 },
593 else => {633 else => {
...@@ -595,32 +635,99 @@ pub const StreamingParser = struct {...@@ -595,32 +635,99 @@ pub const StreamingParser = struct {
595 },635 },
596 },636 },
597637
598 .StringEscapeHexUnicode4 => switch (c) {638 .StringEscapeHexUnicode4 => {
599 '0'...'9', 'A'...'F', 'a'...'f' => {639 var codepoint: u21 = undefined;
600 p.state = .StringEscapeHexUnicode3;640 switch (c) {
601 },641 else => return error.InvalidUnicodeHexSymbol,
602 else => return error.InvalidUnicodeHexSymbol,642 '0'...'9' => {
643 codepoint = c - '0';
644 },
645 'A'...'F' => {
646 codepoint = c - 'A' + 10;
647 },
648 'a'...'f' => {
649 codepoint = c - 'a' + 10;
650 },
651 }
652 p.state = .StringEscapeHexUnicode3;
653 p.string_unicode_codepoint = codepoint << 12;
603 },654 },
604655
605 .StringEscapeHexUnicode3 => switch (c) {656 .StringEscapeHexUnicode3 => {
606 '0'...'9', 'A'...'F', 'a'...'f' => {657 var codepoint: u21 = undefined;
607 p.state = .StringEscapeHexUnicode2;658 switch (c) {
608 },659 else => return error.InvalidUnicodeHexSymbol,
609 else => return error.InvalidUnicodeHexSymbol,660 '0'...'9' => {
661 codepoint = c - '0';
662 },
663 'A'...'F' => {
664 codepoint = c - 'A' + 10;
665 },
666 'a'...'f' => {
667 codepoint = c - 'a' + 10;
668 },
669 }
670 p.state = .StringEscapeHexUnicode2;
671 p.string_unicode_codepoint |= codepoint << 8;
610 },672 },
611673
612 .StringEscapeHexUnicode2 => switch (c) {674 .StringEscapeHexUnicode2 => {
613 '0'...'9', 'A'...'F', 'a'...'f' => {675 var codepoint: u21 = undefined;
614 p.state = .StringEscapeHexUnicode1;676 switch (c) {
615 },677 else => return error.InvalidUnicodeHexSymbol,
616 else => return error.InvalidUnicodeHexSymbol,678 '0'...'9' => {
679 codepoint = c - '0';
680 },
681 'A'...'F' => {
682 codepoint = c - 'A' + 10;
683 },
684 'a'...'f' => {
685 codepoint = c - 'a' + 10;
686 },
687 }
688 p.state = .StringEscapeHexUnicode1;
689 p.string_unicode_codepoint |= codepoint << 4;
617 },690 },
618691
619 .StringEscapeHexUnicode1 => switch (c) {692 .StringEscapeHexUnicode1 => {
620 '0'...'9', 'A'...'F', 'a'...'f' => {693 var codepoint: u21 = undefined;
621 p.state = .String;694 switch (c) {
622 },695 else => return error.InvalidUnicodeHexSymbol,
623 else => return error.InvalidUnicodeHexSymbol,696 '0'...'9' => {
697 codepoint = c - '0';
698 },
699 'A'...'F' => {
700 codepoint = c - 'A' + 10;
701 },
702 'a'...'f' => {
703 codepoint = c - 'a' + 10;
704 },
705 }
706 p.state = .String;
707 p.string_unicode_codepoint |= codepoint;
708 if (p.string_unicode_codepoint < 0xD800 or p.string_unicode_codepoint >= 0xE000) {
709 // not part of surrogate pair
710 p.string_escapes.Some.size_diff -= @as(isize, 6 - (std.unicode.utf8CodepointSequenceLength(p.string_unicode_codepoint) catch unreachable));
711 p.string_last_was_high_surrogate = false;
712 } else if (p.string_unicode_codepoint < 0xDC00) {
713 // 'high' surrogate
714 // takes 3 bytes to encode a half surrogate pair into wtf8
715 p.string_escapes.Some.size_diff -= 6 - 3;
716 p.string_last_was_high_surrogate = true;
717 } else {
718 // 'low' surrogate
719 p.string_escapes.Some.size_diff -= 6;
720 if (p.string_last_was_high_surrogate) {
721 // takes 4 bytes to encode a full surrogate pair into utf8
722 // 3 bytes are already reserved by high surrogate
723 p.string_escapes.Some.size_diff -= -1;
724 } else {
725 // takes 3 bytes to encode a half surrogate pair into wtf8
726 p.string_escapes.Some.size_diff -= -3;
727 }
728 p.string_last_was_high_surrogate = false;
729 }
730 p.string_unicode_codepoint = undefined;
624 },731 },
625732
626 .Number => {733 .Number => {
...@@ -657,6 +764,7 @@ pub const StreamingParser = struct {...@@ -657,6 +764,7 @@ pub const StreamingParser = struct {
657 .is_integer = p.number_is_integer,764 .is_integer = p.number_is_integer,
658 },765 },
659 };766 };
767 p.number_is_integer = undefined;
660 return true;768 return true;
661 },769 },
662 }770 }
...@@ -1271,7 +1379,10 @@ pub const Parser = struct {...@@ -1271,7 +1379,10 @@ pub const Parser = struct {
1271 // TODO: We don't strictly have to copy values which do not contain any escape1379 // TODO: We don't strictly have to copy values which do not contain any escape
1272 // characters if flagged with the option.1380 // characters if flagged with the option.
1273 const slice = s.slice(input, i);1381 const slice = s.slice(input, i);
1274 return Value{ .String = try unescapeStringAlloc(allocator, slice) };1382 return switch (s.escapes) {
1383 .None => Value{ .String = try mem.dupe(allocator, u8, slice) },
1384 .Some => |some_escapes| Value{ .String = try unescapeStringAlloc(allocator, some_escapes, slice) },
1385 };
1275 }1386 }
12761387
1277 fn parseNumber(p: *Parser, n: std.meta.TagPayloadType(Token, Token.Number), input: []const u8, i: usize) !Value {1388 fn parseNumber(p: *Parser, n: std.meta.TagPayloadType(Token, Token.Number), input: []const u8, i: usize) !Value {
...@@ -1285,13 +1396,10 @@ pub const Parser = struct {...@@ -1285,13 +1396,10 @@ pub const Parser = struct {
1285// Unescape a JSON string1396// Unescape a JSON string
1286// Only to be used on strings already validated by the parser1397// Only to be used on strings already validated by the parser
1287// (note the unreachable statements and lack of bounds checking)1398// (note the unreachable statements and lack of bounds checking)
1288// Optimized for arena allocators, uses Allocator.shrink1399fn unescapeStringAlloc(alloc: *Allocator, escapes: std.meta.TagPayloadType(StringEscapes, StringEscapes.Some), input: []const u8) ![]u8 {
1289//1400 const result_size = input.len +% @bitCast(usize, escapes.size_diff);
1290// Idea: count how many bytes we will need to allocate in the streaming parser and store it1401
1291// in the token to avoid allocating too much memory or iterating through the string again1402 const output = try alloc.alloc(u8, result_size);
1292// Downside: need to find how many bytes a unicode escape sequence will produce twice
1293fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1294 const output = try alloc.alloc(u8, input.len);
1295 errdefer alloc.free(output);1403 errdefer alloc.free(output);
12961404
1297 var inIndex: usize = 0;1405 var inIndex: usize = 0;
...@@ -1347,8 +1455,9 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {...@@ -1347,8 +1455,9 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1347 }1455 }
1348 }1456 }
1349 }1457 }
1458 assert(outIndex == result_size);
13501459
1351 return alloc.shrink(output, outIndex);1460 return output;
1352}1461}
13531462
1354test "json.parser.dynamic" {1463test "json.parser.dynamic" {