authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-12-29 18:31:10-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2019-12-29 18:31:10-05:00
log54231e832bae780c5012fc5cd30932447f1e1d47
tree4475e1625e81f320e8ffe358f4ee8dca557569ba
parent6af39aa49afeb3498d6c5dfa0b60a0fdc15ca47c
parent6d3b95a708c87ed81bd4f270b956913d82a0972d
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3648 from xackus/json-unescape

breaking: JSON unescape

2 files changed, 196 insertions(+), 47 deletions(-)

lib/std/json.zig+142-27
......@@ -10,18 +10,18 @@ const maxInt = std.math.maxInt;
1010
1111pub const WriteStream = @import("json/write_stream.zig").WriteStream;
1212
13// A single token slice into the parent string.
14//
15// Use `token.slice()` on the input at the current position to get the current slice.
13/// A single token slice into the parent string.
14///
15/// Use `token.slice()` on the input at the current position to get the current slice.
1616pub const Token = struct {
1717 id: Id,
18 // How many bytes do we skip before counting
18 /// How many bytes do we skip before counting
1919 offset: u1,
20 // Whether string contains a \uXXXX sequence and cannot be zero-copied
20 /// Whether string contains an escape sequence and cannot be zero-copied
2121 string_has_escape: bool,
22 // Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
22 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
2323 number_is_integer: bool,
24 // How many bytes from the current position behind the start of this token is.
24 /// How many bytes from the current position behind the start of this token is.
2525 count: usize,
2626
2727 pub const Id = enum {
......@@ -66,7 +66,7 @@ pub const Token = struct {
6666 };
6767 }
6868
69 // A marker token is a zero-length
69 /// A marker token is a zero-length
7070 pub fn initMarker(id: Id) Token {
7171 return Token{
7272 .id = id,
......@@ -77,19 +77,19 @@ pub const Token = struct {
7777 };
7878 }
7979
80 // Slice into the underlying input string.
80 /// Slice into the underlying input string.
8181 pub fn slice(self: Token, input: []const u8, i: usize) []const u8 {
8282 return input[i + self.offset - self.count .. i + self.offset];
8383 }
8484};
8585
86// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
87// they are encountered. No copies or allocations are performed during parsing and the entire
88// parsing state requires ~40-50 bytes of stack space.
89//
90// Conforms strictly to RFC8529.
91//
92// For a non-byte based wrapper, consider using TokenStream instead.
86/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
87/// they are encountered. No copies or allocations are performed during parsing and the entire
88/// parsing state requires ~40-50 bytes of stack space.
89///
90/// Conforms strictly to RFC8529.
91///
92/// For a non-byte based wrapper, consider using TokenStream instead.
9393pub const StreamingParser = struct {
9494 // Current state
9595 state: State,
......@@ -205,10 +205,10 @@ pub const StreamingParser = struct {
205205 InvalidControlCharacter,
206206 };
207207
208 // Give another byte to the parser and obtain any new tokens. This may (rarely) return two
209 // tokens. token2 is always null if token1 is null.
210 //
211 // There is currently no error recovery on a bad stream.
208 /// Give another byte to the parser and obtain any new tokens. This may (rarely) return two
209 /// tokens. token2 is always null if token1 is null.
210 ///
211 /// There is currently no error recovery on a bad stream.
212212 pub fn feed(p: *StreamingParser, c: u8, token1: *?Token, token2: *?Token) Error!void {
213213 token1.* = null;
214214 token2.* = null;
......@@ -866,7 +866,7 @@ pub const StreamingParser = struct {
866866 }
867867};
868868
869// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.
869/// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.
870870pub const TokenStream = struct {
871871 i: usize,
872872 slice: []const u8,
......@@ -905,7 +905,13 @@ pub const TokenStream = struct {
905905 }
906906 }
907907
908 if (self.parser.complete) {
908 // Without this a bare number fails, becasue the streaming parser doesn't know it ended
909 try self.parser.feed(' ', &t1, &t2);
910 self.i += 1;
911
912 if (t1) |token| {
913 return token;
914 } else if (self.parser.complete) {
909915 return null;
910916 } else {
911917 return error.UnexpectedEndOfJson;
......@@ -971,8 +977,8 @@ test "json.token" {
971977 testing.expect((try p.next()) == null);
972978}
973979
974// Validate a JSON string. This does not limit number precision so a decoder may not necessarily
975// be able to decode the string even if this returns true.
980/// Validate a JSON string. This does not limit number precision so a decoder may not necessarily
981/// be able to decode the string even if this returns true.
976982pub fn validate(s: []const u8) bool {
977983 var p = StreamingParser.init();
978984
......@@ -1009,6 +1015,8 @@ pub const ValueTree = struct {
10091015pub const ObjectMap = StringHashMap(Value);
10101016pub const Array = ArrayList(Value);
10111017
1018/// Represents a JSON value
1019/// Currently only supports numbers that fit into i64 or f64.
10121020pub const Value = union(enum) {
10131021 Null,
10141022 Bool: bool,
......@@ -1055,7 +1063,7 @@ pub const Value = union(enum) {
10551063 }
10561064};
10571065
1058// A non-stream JSON parser which constructs a tree of Value's.
1066/// A non-stream JSON parser which constructs a tree of Value's.
10591067pub const Parser = struct {
10601068 allocator: *Allocator,
10611069 state: State,
......@@ -1124,7 +1132,10 @@ pub const Parser = struct {
11241132 p.state = State.ObjectValue;
11251133 },
11261134 else => {
1127 unreachable;
1135 // The streaming parser would return an error eventually.
1136 // To prevent invalid state we return an error now.
1137 // TODO make the streaming parser return an error as soon as it encounters an invalid object key
1138 return error.InvalidLiteral;
11281139 },
11291140 },
11301141 State.ObjectValue => {
......@@ -1266,7 +1277,7 @@ pub const Parser = struct {
12661277 // TODO: We don't strictly have to copy values which do not contain any escape
12671278 // characters if flagged with the option.
12681279 const slice = token.slice(input, i);
1269 return Value{ .String = try mem.dupe(allocator, u8, slice) };
1280 return Value{ .String = try unescapeStringAlloc(allocator, slice) };
12701281 }
12711282
12721283 fn parseNumber(p: *Parser, token: Token, input: []const u8, i: usize) !Value {
......@@ -1277,6 +1288,77 @@ pub const Parser = struct {
12771288 }
12781289};
12791290
1291// Unescape a JSON string
1292// Only to be used on strings already validated by the parser
1293// (note the unreachable statements and lack of bounds checking)
1294// Optimized for arena allocators, uses Allocator.shrink
1295//
1296// Idea: count how many bytes we will need to allocate in the streaming parser and store it
1297// in the token to avoid allocating too much memory or iterating through the string again
1298// Downside: need to find how many bytes a unicode escape sequence will produce twice
1299fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1300 const output = try alloc.alloc(u8, input.len);
1301 errdefer alloc.free(output);
1302
1303 var inIndex: usize = 0;
1304 var outIndex: usize = 0;
1305
1306 while(inIndex < input.len) {
1307 if(input[inIndex] != '\\'){
1308 // not an escape sequence
1309 output[outIndex] = input[inIndex];
1310 inIndex += 1;
1311 outIndex += 1;
1312 } else if(input[inIndex + 1] != 'u'){
1313 // a simple escape sequence
1314 output[outIndex] = @as(u8,
1315 switch(input[inIndex + 1]){
1316 '\\' => '\\',
1317 '/' => '/',
1318 'n' => '\n',
1319 'r' => '\r',
1320 't' => '\t',
1321 'f' => 12,
1322 'b' => 8,
1323 '"' => '"',
1324 else => unreachable
1325 }
1326 );
1327 inIndex += 2;
1328 outIndex += 1;
1329 } else {
1330 // a unicode escape sequence
1331 const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex+2 .. inIndex+6], 16) catch unreachable;
1332
1333 // guess optimistically that it's not a surrogate pair
1334 if(std.unicode.utf8Encode(firstCodeUnit, output[outIndex..])) |byteCount| {
1335 outIndex += byteCount;
1336 inIndex += 6;
1337 } else |err| {
1338 // it might be a surrogate pair
1339 if(err != error.Utf8CannotEncodeSurrogateHalf) {
1340 return error.InvalidUnicodeHexSymbol;
1341 }
1342 // check if a second code unit is present
1343 if(inIndex + 7 >= input.len or input[inIndex + 6] != '\\' or input[inIndex + 7] != 'u'){
1344 return error.InvalidUnicodeHexSymbol;
1345 }
1346
1347 const secondCodeUnit = std.fmt.parseInt(u16, input[inIndex+8 .. inIndex+12], 16) catch unreachable;
1348
1349 if(std.unicode.utf16leToUtf8(output[outIndex..], [2]u16{ firstCodeUnit, secondCodeUnit })) |byteCount| {
1350 outIndex += byteCount;
1351 inIndex += 12;
1352 } else |_| {
1353 return error.InvalidUnicodeHexSymbol;
1354 }
1355 }
1356 }
1357 }
1358
1359 return alloc.shrink(output, outIndex);
1360}
1361
12801362test "json.parser.dynamic" {
12811363 var p = Parser.init(debug.global_allocator, false);
12821364 defer p.deinit();
......@@ -1399,3 +1481,36 @@ test "integer after float has proper type" {
13991481 );
14001482 std.testing.expect(json.Object.getValue("ints").?.Array.at(0) == .Integer);
14011483}
1484
1485test "escaped characters" {
1486 const input =
1487 \\{
1488 \\ "backslash": "\\",
1489 \\ "forwardslash": "\/",
1490 \\ "newline": "\n",
1491 \\ "carriagereturn": "\r",
1492 \\ "tab": "\t",
1493 \\ "formfeed": "\f",
1494 \\ "backspace": "\b",
1495 \\ "doublequote": "\"",
1496 \\ "unicode": "\u0105",
1497 \\ "surrogatepair": "\ud83d\ude02"
1498 \\}
1499 ;
1500
1501 var p = Parser.init(debug.global_allocator, false);
1502 const tree = try p.parse(input);
1503
1504 const obj = tree.root.Object;
1505
1506 testing.expectEqualSlices(u8, obj.get("backslash").?.value.String, "\\");
1507 testing.expectEqualSlices(u8, obj.get("forwardslash").?.value.String, "/");
1508 testing.expectEqualSlices(u8, obj.get("newline").?.value.String, "\n");
1509 testing.expectEqualSlices(u8, obj.get("carriagereturn").?.value.String, "\r");
1510 testing.expectEqualSlices(u8, obj.get("tab").?.value.String, "\t");
1511 testing.expectEqualSlices(u8, obj.get("formfeed").?.value.String, "\x0C");
1512 testing.expectEqualSlices(u8, obj.get("backspace").?.value.String, "\x08");
1513 testing.expectEqualSlices(u8, obj.get("doublequote").?.value.String, "\"");
1514 testing.expectEqualSlices(u8, obj.get("unicode").?.value.String, "ą");
1515 testing.expectEqualSlices(u8, obj.get("surrogatepair").?.value.String, "😂");
1516}
lib/std/json/test.zig+54-20
......@@ -7,14 +7,46 @@ const std = @import("../std.zig");
77
88fn ok(comptime s: []const u8) void {
99 std.testing.expect(std.json.validate(s));
10
11 var mem_buffer: [1024 * 20]u8 = undefined;
12 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
13 var p = std.json.Parser.init(allocator, false);
14
15 _ = p.parse(s) catch unreachable;
1016}
1117
1218fn err(comptime s: []const u8) void {
1319 std.testing.expect(!std.json.validate(s));
20
21 var mem_buffer: [1024 * 20]u8 = undefined;
22 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
23 var p = std.json.Parser.init(allocator, false);
24
25 if(p.parse(s)) |_| {
26 unreachable;
27 } else |_| {}
1428}
1529
1630fn any(comptime s: []const u8) void {
17 std.testing.expect(true);
31 _ = std.json.validate(s);
32
33 var mem_buffer: [1024 * 20]u8 = undefined;
34 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
35 var p = std.json.Parser.init(allocator, false);
36
37 _ = p.parse(s) catch {};
38}
39
40fn anyStreamingErrNonStreaming(comptime s: []const u8) void {
41 _ = std.json.validate(s);
42
43 var mem_buffer: [1024 * 20]u8 = undefined;
44 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
45 var p = std.json.Parser.init(allocator, false);
46
47 if(p.parse(s)) |_| {
48 unreachable;
49 } else |_| {}
1850}
1951
2052////////////////////////////////////////////////////////////////////////////////////////////////////
......@@ -611,9 +643,9 @@ test "n_array_colon_instead_of_comma" {
611643}
612644
613645test "n_array_comma_after_close" {
614 //err(
615 // \\[""],
616 //);
646 err(
647 \\[""],
648 );
617649}
618650
619651test "n_array_comma_and_number" {
......@@ -641,9 +673,9 @@ test "n_array_extra_close" {
641673}
642674
643675test "n_array_extra_comma" {
644 //err(
645 // \\["",]
646 //);
676 err(
677 \\["",]
678 );
647679}
648680
649681test "n_array_incomplete_invalid_value" {
......@@ -1708,9 +1740,11 @@ test "i_number_double_huge_neg_exp" {
17081740}
17091741
17101742test "i_number_huge_exp" {
1711 any(
1712 \\[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]
1713 );
1743 return error.SkipZigTest;
1744 // FIXME Integer overflow in parseFloat
1745// any(
1746// \\[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]
1747// );
17141748}
17151749
17161750test "i_number_neg_int_huge_exp" {
......@@ -1762,49 +1796,49 @@ test "i_number_very_big_negative_int" {
17621796}
17631797
17641798test "i_object_key_lone_2nd_surrogate" {
1765 any(
1799 anyStreamingErrNonStreaming(
17661800 \\{"\uDFAA":0}
17671801 );
17681802}
17691803
17701804test "i_string_1st_surrogate_but_2nd_missing" {
1771 any(
1805 anyStreamingErrNonStreaming(
17721806 \\["\uDADA"]
17731807 );
17741808}
17751809
17761810test "i_string_1st_valid_surrogate_2nd_invalid" {
1777 any(
1811 anyStreamingErrNonStreaming(
17781812 \\["\uD888\u1234"]
17791813 );
17801814}
17811815
17821816test "i_string_incomplete_surrogate_and_escape_valid" {
1783 any(
1817 anyStreamingErrNonStreaming(
17841818 \\["\uD800\n"]
17851819 );
17861820}
17871821
17881822test "i_string_incomplete_surrogate_pair" {
1789 any(
1823 anyStreamingErrNonStreaming(
17901824 \\["\uDd1ea"]
17911825 );
17921826}
17931827
17941828test "i_string_incomplete_surrogates_escape_valid" {
1795 any(
1829 anyStreamingErrNonStreaming(
17961830 \\["\uD800\uD800\n"]
17971831 );
17981832}
17991833
18001834test "i_string_invalid_lonely_surrogate" {
1801 any(
1835 anyStreamingErrNonStreaming(
18021836 \\["\ud800"]
18031837 );
18041838}
18051839
18061840test "i_string_invalid_surrogate" {
1807 any(
1841 anyStreamingErrNonStreaming(
18081842 \\["\ud800abc"]
18091843 );
18101844}
......@@ -1816,7 +1850,7 @@ test "i_string_invalid_utf-8" {
18161850}
18171851
18181852test "i_string_inverted_surrogates_U+1D11E" {
1819 any(
1853 anyStreamingErrNonStreaming(
18201854 \\["\uDd1e\uD834"]
18211855 );
18221856}
......@@ -1828,7 +1862,7 @@ test "i_string_iso_latin_1" {
18281862}
18291863
18301864test "i_string_lone_second_surrogate" {
1831 any(
1865 anyStreamingErrNonStreaming(
18321866 \\["\uDFAA"]
18331867 );
18341868}