| ... | ... | @@ -10,18 +10,18 @@ const maxInt = std.math.maxInt; |
| 10 | 10 | |
| 11 | 11 | pub const WriteStream = @import("json/write_stream.zig").WriteStream; |
| 12 | 12 | |
| 13 | | // A single token slice into the parent string. |
| 14 | | // |
| 15 | | // Use `token.slice()` on the input at the current position to get the current slice. |
| 13 | /// A single token slice into the parent string. |
| 14 | /// |
| 15 | /// Use `token.slice()` on the input at the current position to get the current slice. |
| 16 | 16 | pub const Token = struct { |
| 17 | 17 | id: Id, |
| 18 | | // How many bytes do we skip before counting |
| 18 | /// How many bytes do we skip before counting |
| 19 | 19 | offset: u1, |
| 20 | | // Whether string contains a \uXXXX sequence and cannot be zero-copied |
| 20 | /// Whether string contains an escape sequence and cannot be zero-copied |
| 21 | 21 | string_has_escape: bool, |
| 22 | | // Whether number is simple and can be represented by an integer (i.e. no `.` or `e`) |
| 22 | /// Whether number is simple and can be represented by an integer (i.e. no `.` or `e`) |
| 23 | 23 | number_is_integer: bool, |
| 24 | | // How many bytes from the current position behind the start of this token is. |
| 24 | /// How many bytes from the current position behind the start of this token is. |
| 25 | 25 | count: usize, |
| 26 | 26 | |
| 27 | 27 | pub const Id = enum { |
| ... | ... | @@ -66,7 +66,7 @@ pub const Token = struct { |
| 66 | 66 | }; |
| 67 | 67 | } |
| 68 | 68 | |
| 69 | | // A marker token is a zero-length |
| 69 | /// A marker token is a zero-length |
| 70 | 70 | pub fn initMarker(id: Id) Token { |
| 71 | 71 | return Token{ |
| 72 | 72 | .id = id, |
| ... | ... | @@ -77,19 +77,19 @@ pub const Token = struct { |
| 77 | 77 | }; |
| 78 | 78 | } |
| 79 | 79 | |
| 80 | | // Slice into the underlying input string. |
| 80 | /// Slice into the underlying input string. |
| 81 | 81 | pub fn slice(self: Token, input: []const u8, i: usize) []const u8 { |
| 82 | 82 | return input[i + self.offset - self.count .. i + self.offset]; |
| 83 | 83 | } |
| 84 | 84 | }; |
| 85 | 85 | |
| 86 | | // A small streaming JSON parser. This accepts input one byte at a time and returns tokens as |
| 87 | | // they are encountered. No copies or allocations are performed during parsing and the entire |
| 88 | | // parsing state requires ~40-50 bytes of stack space. |
| 89 | | // |
| 90 | | // Conforms strictly to RFC8529. |
| 91 | | // |
| 92 | | // For a non-byte based wrapper, consider using TokenStream instead. |
| 86 | /// A small streaming JSON parser. This accepts input one byte at a time and returns tokens as |
| 87 | /// they are encountered. No copies or allocations are performed during parsing and the entire |
| 88 | /// parsing state requires ~40-50 bytes of stack space. |
| 89 | /// |
| 90 | /// Conforms strictly to RFC8529. |
| 91 | /// |
| 92 | /// For a non-byte based wrapper, consider using TokenStream instead. |
| 93 | 93 | pub const StreamingParser = struct { |
| 94 | 94 | // Current state |
| 95 | 95 | state: State, |
| ... | ... | @@ -205,10 +205,10 @@ pub const StreamingParser = struct { |
| 205 | 205 | InvalidControlCharacter, |
| 206 | 206 | }; |
| 207 | 207 | |
| 208 | | // Give another byte to the parser and obtain any new tokens. This may (rarely) return two |
| 209 | | // tokens. token2 is always null if token1 is null. |
| 210 | | // |
| 211 | | // There is currently no error recovery on a bad stream. |
| 208 | /// Give another byte to the parser and obtain any new tokens. This may (rarely) return two |
| 209 | /// tokens. token2 is always null if token1 is null. |
| 210 | /// |
| 211 | /// There is currently no error recovery on a bad stream. |
| 212 | 212 | pub fn feed(p: *StreamingParser, c: u8, token1: *?Token, token2: *?Token) Error!void { |
| 213 | 213 | token1.* = null; |
| 214 | 214 | token2.* = null; |
| ... | ... | @@ -860,7 +860,7 @@ pub const StreamingParser = struct { |
| 860 | 860 | } |
| 861 | 861 | }; |
| 862 | 862 | |
| 863 | | // A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens. |
| 863 | /// A small wrapper over a StreamingParser for full slices. Returns a stream of json Tokens. |
| 864 | 864 | pub const TokenStream = struct { |
| 865 | 865 | i: usize, |
| 866 | 866 | slice: []const u8, |
| ... | ... | @@ -898,7 +898,13 @@ pub const TokenStream = struct { |
| 898 | 898 | } |
| 899 | 899 | } |
| 900 | 900 | |
| 901 | | if (self.parser.complete) { |
| 901 | // Without this a bare number fails, becasue the streaming parser doesn't know it ended |
| 902 | try self.parser.feed(' ', &t1, &t2); |
| 903 | self.i += 1; |
| 904 | |
| 905 | if (t1) |token| { |
| 906 | return token; |
| 907 | } else if (self.parser.complete) { |
| 902 | 908 | return null; |
| 903 | 909 | } else { |
| 904 | 910 | return error.UnexpectedEndOfJson; |
| ... | ... | @@ -1050,7 +1056,7 @@ pub const Value = union(enum) { |
| 1050 | 1056 | } |
| 1051 | 1057 | }; |
| 1052 | 1058 | |
| 1053 | | // A non-stream JSON parser which constructs a tree of Value's. |
| 1059 | /// A non-stream JSON parser which constructs a tree of Value's. |
| 1054 | 1060 | pub const Parser = struct { |
| 1055 | 1061 | allocator: *Allocator, |
| 1056 | 1062 | state: State, |
| ... | ... | @@ -1119,7 +1125,10 @@ pub const Parser = struct { |
| 1119 | 1125 | p.state = State.ObjectValue; |
| 1120 | 1126 | }, |
| 1121 | 1127 | else => { |
| 1122 | | unreachable; |
| 1128 | // The streaming parser would return an error eventually. |
| 1129 | // To prevent invalid state we return an error now. |
| 1130 | // TODO make the streaming parser return an error as soon as it encounters an invalid object key |
| 1131 | return error.InvalidLiteral; |
| 1123 | 1132 | }, |
| 1124 | 1133 | }, |
| 1125 | 1134 | State.ObjectValue => { |
| ... | ... | @@ -1276,6 +1285,10 @@ pub const Parser = struct { |
| 1276 | 1285 | // Only to be used on strings already validated by the parser |
| 1277 | 1286 | // (note the unreachable statements and lack of bounds checking) |
| 1278 | 1287 | // Optimized for arena allocators, uses Allocator.shrink |
| 1288 | // |
| 1289 | // Idea: count how many bytes we will need to allocate in the streaming parser and store it |
| 1290 | // in the token to avoid allocating too much memory or iterating through the string again |
| 1291 | // Downside: need to find how many bytes a unicode escape sequence will produce twice |
| 1279 | 1292 | fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 { |
| 1280 | 1293 | const output = try alloc.alloc(u8, input.len); |
| 1281 | 1294 | errdefer alloc.free(output); |
| ... | ... | @@ -1290,22 +1303,22 @@ fn unescapeStringAlloc(alloc: *Allocator, input: []const u8) ![]u8 { |
| 1290 | 1303 | inIndex += 1; |
| 1291 | 1304 | outIndex += 1; |
| 1292 | 1305 | } else if(input[inIndex + 1] != 'u'){ |
| 1293 | | // a simple escape sequence |
| 1294 | | output[outIndex] = @as(u8, |
| 1295 | | switch(input[inIndex + 1]){ |
| 1296 | | '\\' => '\\', |
| 1297 | | '/' => '/', |
| 1298 | | 'n' => '\n', |
| 1299 | | 'r' => '\r', |
| 1300 | | 't' => '\t', |
| 1301 | | 'f' => 12, |
| 1302 | | 'b' => 8, |
| 1303 | | '"' => '"', |
| 1304 | | else => unreachable |
| 1305 | | } |
| 1306 | | ); |
| 1307 | | inIndex += 2; |
| 1308 | | outIndex += 1; |
| 1306 | // a simple escape sequence |
| 1307 | output[outIndex] = @as(u8, |
| 1308 | switch(input[inIndex + 1]){ |
| 1309 | '\\' => '\\', |
| 1310 | '/' => '/', |
| 1311 | 'n' => '\n', |
| 1312 | 'r' => '\r', |
| 1313 | 't' => '\t', |
| 1314 | 'f' => 12, |
| 1315 | 'b' => 8, |
| 1316 | '"' => '"', |
| 1317 | else => unreachable |
| 1318 | } |
| 1319 | ); |
| 1320 | inIndex += 2; |
| 1321 | outIndex += 1; |
| 1309 | 1322 | } else { |
| 1310 | 1323 | // a unicode escape sequence |
| 1311 | 1324 | const firstCodeUnit = std.fmt.parseInt(u16, input[inIndex+2 .. inIndex+6], 16) catch unreachable; |
xackus <14938807+xackus@users.noreply.github.com> 2019-11-11 23:25:54+01:00