authorgravatar for 14938807+xackus@users.noreply.github.comxackus <14938807+xackus@users.noreply.github.com> 2019-11-11 23:25:54+01:00
committergravatar for 14938807+xackus@users.noreply.github.comxackus <14938807+xackus@users.noreply.github.com> 2019-11-11 23:25:54+01:00
logf9b7d6d75d24fa5844a6b83be312722d1a71efaf
tree73ad9202faeb6a3ad2175bc90a780b22325facdb
parent371747d8fb270c7d2f80a5e3a43ef0485332a070

Fix bugs in JSON parser

Make comments into documentation where appropriate

2 files changed, 69 insertions(+), 60 deletions(-)

lib/std/json.zig+53-40
...@@ -10,18 +10,18 @@ const maxInt = std.math.maxInt;...@@ -10,18 +10,18 @@ const maxInt = std.math.maxInt;
1010
11pub const WriteStream = @import("json/write_stream.zig").WriteStream;11pub const WriteStream = @import("json/write_stream.zig").WriteStream;
1212
13// A single token slice into the parent string.13/// A single token slice into the parent string.
14//14///
15// Use `token.slice()` on the input at the current position to get the current slice.15/// Use `token.slice()` on the input at the current position to get the current slice.
16pub const Token = struct {16pub const Token = struct {
17 id: Id,17 id: Id,
18 // How many bytes do we skip before counting18 /// How many bytes do we skip before counting
19 offset: u1,19 offset: u1,
20 // Whether string contains a \uXXXX sequence and cannot be zero-copied20 /// Whether string contains an escape sequence and cannot be zero-copied
21 string_has_escape: bool,21 string_has_escape: bool,
22 // Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)22 /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`)
23 number_is_integer: bool,23 number_is_integer: bool,
24 // How many bytes from the current position behind the start of this token is.24 /// How many bytes from the current position behind the start of this token is.
25 count: usize,25 count: usize,
2626
27 pub const Id = enum {27 pub const Id = enum {
...@@ -66,7 +66,7 @@ pub const Token = struct {...@@ -66,7 +66,7 @@ pub const Token = struct {
66 };66 };
67 }67 }
6868
69 // A marker token is a zero-length69 /// A marker token is a zero-length
70 pub fn initMarker(id: Id) Token {70 pub fn initMarker(id: Id) Token {
71 return Token{71 return Token{
72 .id = id,72 .id = id,
...@@ -77,19 +77,19 @@ pub const Token = struct {...@@ -77,19 +77,19 @@ pub const Token = struct {
77 };77 };
78 }78 }
7979
80 // Slice into the underlying input string.80 /// Slice into the underlying input string.
81 pub fn slice(self: Token, input: []const u8, i: usize) []const u8 {81 pub fn slice(self: Token, input: []const u8, i: usize) []const u8 {
82 return input[i + self.offset - self.count .. i + self.offset];82 return input[i + self.offset - self.count .. i + self.offset];
83 }83 }
84};84};
8585
86// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as86/// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as
87// they are encountered. No copies or allocations are performed during parsing and the entire87/// they are encountered. No copies or allocations are performed during parsing and the entire
88// parsing state requires ~40-50 bytes of stack space.88/// parsing state requires ~40-50 bytes of stack space.
89//89///
90// Conforms strictly to RFC8529.90/// Conforms strictly to RFC8529.
91//91///
92// For a non-byte based wrapper, consider using TokenStream instead.92/// For a non-byte based wrapper, consider using TokenStream instead.
93pub const StreamingParser = struct {93pub const StreamingParser = struct {
94 // Current state94 // Current state
95 state: State,95 state: State,
...@@ -205,10 +205,10 @@ pub const StreamingParser = struct {...@@ -205,10 +205,10 @@ pub const StreamingParser = struct {
205 InvalidControlCharacter,205 InvalidControlCharacter,
206 };206 };
207207
208 // Give another byte to the parser and obtain any new tokens. This may (rarely) return two208 /// Give another byte to the parser and obtain any new tokens. This may (rarely) return two
209 // tokens. token2 is always null if token1 is null.209 /// tokens. token2 is always null if token1 is null.
210 //210 ///
211 // There is currently no error recovery on a bad stream.211 /// There is currently no error recovery on a bad stream.
212 pub fn feed(p: *StreamingParser, c: u8, token1: *?Token, token2: *?Token) Error!void {212 pub fn feed(p: *StreamingParser, c: u8, token1: *?Token, token2: *?Token) Error!void {
213 token1.* = null;213 token1.* = null;
214 token2.* = null;214 token2.* = null;
...@@ -860,7 +860,7 @@ pub const StreamingParser = struct {...@@ -860,7 +860,7 @@ pub const StreamingParser = struct {
860 }860 }
861};861};
862862
863// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.863/// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens.
864pub const TokenStream = struct {864pub const TokenStream = struct {
865 i: usize,865 i: usize,
866 slice: []const u8,866 slice: []const u8,
...@@ -898,7 +898,13 @@ pub const TokenStream = struct {...@@ -898,7 +898,13 @@ pub const TokenStream = struct {
898 }898 }
899 }899 }
900900
901 if (self.parser.complete) {901 // Without this a bare number fails, becasue the streaming parser doesn't know it ended
902 try self.parser.feed(' ', &t1, &t2);
903 self.i += 1;
904
905 if (t1) |token| {
906 return token;
907 } else if (self.parser.complete) {
902 return null;908 return null;
903 } else {909 } else {
904 return error.UnexpectedEndOfJson;910 return error.UnexpectedEndOfJson;
...@@ -1050,7 +1056,7 @@ pub const Value = union(enum) {...@@ -1050,7 +1056,7 @@ pub const Value = union(enum) {
1050 }1056 }
1051};1057};
10521058
1053// A non-stream JSON parser which constructs a tree of Value's.1059/// A non-stream JSON parser which constructs a tree of Value's.
1054pub const Parser = struct {1060pub const Parser = struct {
1055 allocator: *Allocator,1061 allocator: *Allocator,
1056 state: State,1062 state: State,
...@@ -1119,7 +1125,10 @@ pub const Parser = struct {...@@ -1119,7 +1125,10 @@ pub const Parser = struct {
1119 p.state = State.ObjectValue;1125 p.state = State.ObjectValue;
1120 },1126 },
1121 else => {1127 else => {
1122 unreachable;1128 // The streaming parser would return an error eventually.
1129 // To prevent invalid state we return an error now.
1130 // TODO make the streaming parser return an error as soon as it encounters an invalid object key
1131 return error.InvalidLiteral;
1123 },1132 },
1124 },1133 },
1125 State.ObjectValue => {1134 State.ObjectValue => {
...@@ -1276,6 +1285,10 @@ pub const Parser = struct {...@@ -1276,6 +1285,10 @@ pub const Parser = struct {
1276// Only to be used on strings already validated by the parser1285// Only to be used on strings already validated by the parser
1277// (note the unreachable statements and lack of bounds checking)1286// (note the unreachable statements and lack of bounds checking)
1278// Optimized for arena allocators, uses Allocator.shrink1287// Optimized for arena allocators, uses Allocator.shrink
1288//
1289// Idea: count how many bytes we will need to allocate in the streaming parser and store it
1290// in the token to avoid allocating too much memory or iterating through the string again
1291// Downside: need to find how many bytes a unicode escape sequence will produce twice
1279fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {1292fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1280 const output = try alloc.alloc(u8, input.len);1293 const output = try alloc.alloc(u8, input.len);
1281 errdefer alloc.free(output);1294 errdefer alloc.free(output);
...@@ -1290,22 +1303,22 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {...@@ -1290,22 +1303,22 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 {
1290 inIndex += 1;1303 inIndex += 1;
1291 outIndex += 1;1304 outIndex += 1;
1292 } else if(input[inIndex + 1] != 'u'){1305 } else if(input[inIndex + 1] != 'u'){
1293 // a simple escape sequence1306 // a simple escape sequence
1294 output[outIndex] = @as(u8,1307 output[outIndex] = @as(u8,
1295 switch(input[inIndex + 1]){1308 switch(input[inIndex + 1]){
1296 '\\' => '\\',1309 '\\' => '\\',
1297 '/' => '/',1310 '/' => '/',
1298 'n' => '\n',1311 'n' => '\n',
1299 'r' => '\r',1312 'r' => '\r',
1300 't' => '\t',1313 't' => '\t',
1301 'f' => 12,1314 'f' => 12,
1302 'b' => 8,1315 'b' => 8,
1303 '"' => '"',1316 '"' => '"',
1304 else => unreachable1317 else => unreachable
1305 }1318 }
1306 );1319 );
1307 inIndex += 2;1320 inIndex += 2;
1308 outIndex += 1;1321 outIndex += 1;
1309 } else {1322 } else {
1310 // a unicode escape sequence1323 // a unicode escape sequence
1311 const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex+2 .. inIndex+6], 16) catch unreachable;1324 const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex+2 .. inIndex+6], 16) catch unreachable;
lib/std/json/test.zig+16-20
...@@ -559,17 +559,15 @@ test "y_structure_lonely_false" {...@@ -559,17 +559,15 @@ test "y_structure_lonely_false" {
559}559}
560560
561test "y_structure_lonely_int" {561test "y_structure_lonely_int" {
562 return error.SkipZigTest;562 ok(
563// ok(563 \\42
564// \\42564 );
565// );
566}565}
567566
568test "y_structure_lonely_negative_real" {567test "y_structure_lonely_negative_real" {
569 return error.SkipZigTest;568 ok(
570// ok(569 \\-0.1
571// \\-0.1570 );
572// );
573}571}
574572
575test "y_structure_lonely_null" {573test "y_structure_lonely_null" {
...@@ -1107,10 +1105,9 @@ test "n_object_bad_value" {...@@ -1107,10 +1105,9 @@ test "n_object_bad_value" {
1107}1105}
11081106
1109test "n_object_bracket_key" {1107test "n_object_bracket_key" {
1110 return error.SkipZigTest;1108 err(
1111// err(1109 \\{[: "x"}
1112// \\{[: "x"}1110 );
1113// );
1114}1111}
11151112
1116test "n_object_comma_instead_of_colon" {1113test "n_object_comma_instead_of_colon" {
...@@ -1192,10 +1189,9 @@ test "n_object_non_string_key" {...@@ -1192,10 +1189,9 @@ test "n_object_non_string_key" {
1192}1189}
11931190
1194test "n_object_repeated_null_null" {1191test "n_object_repeated_null_null" {
1195 return error.SkipZigTest;1192 err(
1196// err(1193 \\{null:null,null:null}
1197// \\{null:null,null:null}1194 );
1198// );
1199}1195}
12001196
1201test "n_object_several_trailing_commas" {1197test "n_object_several_trailing_commas" {
...@@ -1618,10 +1614,9 @@ test "n_structure_open_object" {...@@ -1618,10 +1614,9 @@ test "n_structure_open_object" {
1618}1614}
16191615
1620test "n_structure_open_object_open_array" {1616test "n_structure_open_object_open_array" {
1621 return error.SkipZigTest;1617 err(
1622 // err(1618 \\{[
1623 // \\{[1619 );
1624 // );
1625}1620}
16261621
1627test "n_structure_open_object_open_string" {1622test "n_structure_open_object_open_string" {
...@@ -1734,6 +1729,7 @@ test "i_number_double_huge_neg_exp" {...@@ -1734,6 +1729,7 @@ test "i_number_double_huge_neg_exp" {
17341729
1735test "i_number_huge_exp" {1730test "i_number_huge_exp" {
1736 return error.SkipZigTest;1731 return error.SkipZigTest;
1732 // FIXME Integer overflow in parseFloat
1737// any(1733// any(
1738// \\[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]1734// \\[0.4e00669999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999969999999006]
1739// );1735// );