authorgravatar for garrettlennoxbeck@gmail.comGarrett Beck <garrettlennoxbeck@gmail.com> 2023-07-08 21:49:31-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-07-08 22:49:31-04:00
log131bfe2f7488cab02c492ca3afc6d2a18bc83aac
tree55d953dcff09a9f3341eaba8a81a576b2ea81fb8
parent89396ff02ba235592641fb388e3958c2c047e728
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.json: expose innerParse and add .allocate option (#16312)

3 files changed, 145 insertions(+), 19 deletions(-)

lib/std/json.zig+1
......@@ -88,6 +88,7 @@ pub const parseFromSlice = @import("json/static.zig").parseFromSlice;
8888pub const parseFromSliceLeaky = @import("json/static.zig").parseFromSliceLeaky;
8989pub const parseFromTokenSource = @import("json/static.zig").parseFromTokenSource;
9090pub const parseFromTokenSourceLeaky = @import("json/static.zig").parseFromTokenSourceLeaky;
91pub const innerParse = @import("json/static.zig").innerParse;
9192pub const parseFromValue = @import("json/static.zig").parseFromValue;
9293pub const parseFromValueLeaky = @import("json/static.zig").parseFromValueLeaky;
9394pub const ParseError = @import("json/static.zig").ParseError;
lib/std/json/static.zig+44-19
......@@ -34,6 +34,14 @@ pub const ParseOptions = struct {
3434 /// The default for `parseFromTokenSource` with a `*std.json.Reader` is `std.json.default_max_value_len`.
3535 /// Ignored for `parseFromValue` and `parseFromValueLeaky`.
3636 max_value_len: ?usize = null,
37
38 /// This determines whether strings should always be copied,
39 /// or if a reference to the given buffer should be preferred if possible.
40 /// The default for `parseFromSlice` or `parseFromTokenSource` with a `*std.json.Scanner` input
41 /// is `.alloc_if_needed`.
42 /// The default with a `*std.json.Reader` input is `.alloc_always`.
43 /// Ignored for `parseFromValue` and `parseFromValueLeaky`.
44 allocate: ?AllocWhen = null,
3745};
3846
3947pub fn Parsed(comptime T: type) type {
......@@ -113,7 +121,6 @@ pub fn parseFromTokenSourceLeaky(
113121 if (@TypeOf(scanner_or_reader.*) == Scanner) {
114122 assert(scanner_or_reader.is_end_of_input);
115123 }
116
117124 var resolved_options = options;
118125 if (resolved_options.max_value_len == null) {
119126 if (@TypeOf(scanner_or_reader.*) == Scanner) {
......@@ -122,8 +129,15 @@ pub fn parseFromTokenSourceLeaky(
122129 resolved_options.max_value_len = default_max_value_len;
123130 }
124131 }
132 if (resolved_options.allocate == null) {
133 if (@TypeOf(scanner_or_reader.*) == Scanner) {
134 resolved_options.allocate = .alloc_if_needed;
135 } else {
136 resolved_options.allocate = .alloc_always;
137 }
138 }
125139
126 const value = try internalParse(T, allocator, scanner_or_reader, resolved_options);
140 const value = try innerParse(T, allocator, scanner_or_reader, resolved_options);
127141
128142 assert(.end_of_document == try scanner_or_reader.next());
129143
......@@ -181,7 +195,14 @@ pub const ParseFromValueError = std.fmt.ParseIntError || std.fmt.ParseFloatError
181195 LengthMismatch,
182196};
183197
184fn internalParse(
198/// This is an internal function called recursively
199/// during the implementation of `parseFromTokenSourceLeaky` and similar.
200/// It is exposed primarily to enable custom `jsonParse()` methods to call back into the `parseFrom*` system,
201/// such as if you're implementing a custom container of type `T`;
202/// you can call `internalParse(T, ...)` for each of the container's items.
203/// Note that `null` fields are not allowed on the `options` when calling this function.
204/// (The `options` you get in your `jsonParse` method has no `null` fields.)
205pub fn innerParse(
185206 comptime T: type,
186207 allocator: Allocator,
187208 source: anytype,
......@@ -220,7 +241,7 @@ fn internalParse(
220241 return null;
221242 },
222243 else => {
223 return try internalParse(optionalInfo.child, allocator, source, options);
244 return try innerParse(optionalInfo.child, allocator, source, options);
224245 },
225246 }
226247 },
......@@ -250,16 +271,17 @@ fn internalParse(
250271 var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
251272 const field_name = switch (name_token.?) {
252273 inline .string, .allocated_string => |slice| slice,
253 else => return error.UnexpectedToken,
274 else => {
275 return error.UnexpectedToken;
276 },
254277 };
255278
256279 inline for (unionInfo.fields) |u_field| {
257280 if (std.mem.eql(u8, u_field.name, field_name)) {
258281 // Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
259 // (Recursing into internalParse() might trigger more allocations.)
282 // (Recursing into innerParse() might trigger more allocations.)
260283 freeAllocated(allocator, name_token.?);
261284 name_token = null;
262
263285 if (u_field.type == void) {
264286 // void isn't really a json type, but we can support void payload union tags with {} as a value.
265287 if (.object_begin != try source.next()) return error.UnexpectedToken;
......@@ -267,7 +289,7 @@ fn internalParse(
267289 result = @unionInit(T, u_field.name, {});
268290 } else {
269291 // Recurse.
270 result = @unionInit(T, u_field.name, try internalParse(u_field.type, allocator, source, options));
292 result = @unionInit(T, u_field.name, try innerParse(u_field.type, allocator, source, options));
271293 }
272294 break;
273295 }
......@@ -287,7 +309,7 @@ fn internalParse(
287309
288310 var r: T = undefined;
289311 inline for (0..structInfo.fields.len) |i| {
290 r[i] = try internalParse(structInfo.fields[i].type, allocator, source, options);
312 r[i] = try innerParse(structInfo.fields[i].type, allocator, source, options);
291313 }
292314
293315 if (.array_end != try source.next()) return error.UnexpectedToken;
......@@ -307,32 +329,35 @@ fn internalParse(
307329 while (true) {
308330 var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
309331 const field_name = switch (name_token.?) {
310 .object_end => break, // No more fields.
311332 inline .string, .allocated_string => |slice| slice,
312 else => return error.UnexpectedToken,
333 .object_end => { // No more fields.
334 break;
335 },
336 else => {
337 return error.UnexpectedToken;
338 },
313339 };
314340
315341 inline for (structInfo.fields, 0..) |field, i| {
316342 if (field.is_comptime) @compileError("comptime fields are not supported: " ++ @typeName(T) ++ "." ++ field.name);
317343 if (std.mem.eql(u8, field.name, field_name)) {
318344 // Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
319 // (Recursing into internalParse() might trigger more allocations.)
345 // (Recursing into innerParse() might trigger more allocations.)
320346 freeAllocated(allocator, name_token.?);
321347 name_token = null;
322
323348 if (fields_seen[i]) {
324349 switch (options.duplicate_field_behavior) {
325350 .use_first => {
326351 // Parse and ignore the redundant value.
327352 // We don't want to skip the value, because we want type checking.
328 _ = try internalParse(field.type, allocator, source, options);
353 _ = try innerParse(field.type, allocator, source, options);
329354 break;
330355 },
331356 .@"error" => return error.DuplicateField,
332357 .use_last => {},
333358 }
334359 }
335 @field(r, field.name) = try internalParse(field.type, allocator, source, options);
360 @field(r, field.name) = try innerParse(field.type, allocator, source, options);
336361 fields_seen[i] = true;
337362 break;
338363 }
......@@ -418,7 +443,7 @@ fn internalParse(
418443 switch (ptrInfo.size) {
419444 .One => {
420445 const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
421 r.* = try internalParse(ptrInfo.child, allocator, source, options);
446 r.* = try innerParse(ptrInfo.child, allocator, source, options);
422447 return r;
423448 },
424449 .Slice => {
......@@ -438,7 +463,7 @@ fn internalParse(
438463 }
439464
440465 try arraylist.ensureUnusedCapacity(1);
441 arraylist.appendAssumeCapacity(try internalParse(ptrInfo.child, allocator, source, options));
466 arraylist.appendAssumeCapacity(try innerParse(ptrInfo.child, allocator, source, options));
442467 }
443468
444469 if (ptrInfo.sentinel) |some| {
......@@ -459,7 +484,7 @@ fn internalParse(
459484 return try value_list.toOwnedSliceSentinel(@as(*const u8, @ptrCast(sentinel_ptr)).*);
460485 }
461486 if (ptrInfo.is_const) {
462 switch (try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?)) {
487 switch (try source.nextAllocMax(allocator, options.allocate.?, options.max_value_len.?)) {
463488 inline .string, .allocated_string => |slice| return slice,
464489 else => unreachable,
465490 }
......@@ -495,7 +520,7 @@ fn internalParseArray(
495520 var r: T = undefined;
496521 var i: usize = 0;
497522 while (i < len) : (i += 1) {
498 r[i] = try internalParse(Child, allocator, source, options);
523 r[i] = try innerParse(Child, allocator, source, options);
499524 }
500525
501526 if (.array_end != try source.next()) return error.UnexpectedToken;
lib/std/json/static_test.zig+100
......@@ -7,6 +7,7 @@ const parseFromSlice = @import("./static.zig").parseFromSlice;
77const parseFromSliceLeaky = @import("./static.zig").parseFromSliceLeaky;
88const parseFromTokenSource = @import("./static.zig").parseFromTokenSource;
99const parseFromTokenSourceLeaky = @import("./static.zig").parseFromTokenSourceLeaky;
10const innerParse = @import("./static.zig").innerParse;
1011const parseFromValue = @import("./static.zig").parseFromValue;
1112const parseFromValueLeaky = @import("./static.zig").parseFromValueLeaky;
1213const ParseOptions = @import("./static.zig").ParseOptions;
......@@ -801,3 +802,102 @@ test "parse into vector" {
801802 try testing.expectApproxEqAbs(@as(f32, 2.5), parsed.value.vec_f32[1], 0.0000001);
802803 try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, parsed.value.vec_i32);
803804}
805
806fn assertKey(
807 allocator: Allocator,
808 test_string: []const u8,
809 scanner: anytype,
810) !void {
811 const token_outer = try scanner.nextAlloc(allocator, .alloc_always);
812 switch (token_outer) {
813 .allocated_string => |string| {
814 try testing.expectEqualSlices(u8, string, test_string);
815 allocator.free(string);
816 },
817 else => return error.UnexpectedToken,
818 }
819}
820test "json parse partial" {
821 const Inner = struct {
822 num: u32,
823 yes: bool,
824 };
825 var str =
826 \\{
827 \\ "outer": {
828 \\ "key1": {
829 \\ "num": 75,
830 \\ "yes": true
831 \\ },
832 \\ "key2": {
833 \\ "num": 95,
834 \\ "yes": false
835 \\ }
836 \\ }
837 \\}
838 ;
839 var allocator = testing.allocator;
840 var scanner = JsonScanner.initCompleteInput(allocator, str);
841 defer scanner.deinit();
842
843 var arena = ArenaAllocator.init(allocator);
844 defer arena.deinit();
845
846 // Peel off the outer object
847 try testing.expectEqual(try scanner.next(), .object_begin);
848 try assertKey(allocator, "outer", &scanner);
849 try testing.expectEqual(try scanner.next(), .object_begin);
850 try assertKey(allocator, "key1", &scanner);
851
852 // Parse the inner object to an Inner struct
853 const inner_token = try innerParse(
854 Inner,
855 arena.allocator(),
856 &scanner,
857 .{ .max_value_len = scanner.input.len },
858 );
859 try testing.expectEqual(inner_token.num, 75);
860 try testing.expectEqual(inner_token.yes, true);
861
862 // Get they next key
863 try assertKey(allocator, "key2", &scanner);
864 const inner_token_2 = try innerParse(
865 Inner,
866 arena.allocator(),
867 &scanner,
868 .{ .max_value_len = scanner.input.len },
869 );
870 try testing.expectEqual(inner_token_2.num, 95);
871 try testing.expectEqual(inner_token_2.yes, false);
872 try testing.expectEqual(try scanner.next(), .object_end);
873}
874
875test "json parse allocate when streaming" {
876 const T = struct {
877 not_const: []u8,
878 is_const: []const u8,
879 };
880 var str =
881 \\{
882 \\ "not_const": "non const string",
883 \\ "is_const": "const string"
884 \\}
885 ;
886 var allocator = testing.allocator;
887 var arena = ArenaAllocator.init(allocator);
888 defer arena.deinit();
889
890 var stream = std.io.fixedBufferStream(str);
891 var json_reader = jsonReader(std.testing.allocator, stream.reader());
892
893 const parsed = parseFromTokenSourceLeaky(T, arena.allocator(), &json_reader, .{}) catch |err| {
894 json_reader.deinit();
895 return err;
896 };
897 // Deinit our reader to invalidate its buffer
898 json_reader.deinit();
899
900 // If either of these was invalidated, it would be full of '0xAA'
901 try testing.expectEqualSlices(u8, parsed.not_const, "non const string");
902 try testing.expectEqualSlices(u8, parsed.is_const, "const string");
903}