Commit 131bfe2f7488cab02c492ca3afc6d2a18bc83aac | zig

author	Garrett Beck <garrettlennoxbeck@gmail.com> 2023-07-08 21:49:31-05:00
committer	GitHub <noreply@github.com> 2023-07-08 22:49:31-04:00
log	131bfe2f7488cab02c492ca3afc6d2a18bc83aac
tree	55d953dcff09a9f3341eaba8a81a576b2ea81fb8
parent	89396ff02ba235592641fb388e3958c2c047e728
signature	Signed by PGP key 4AEE18F83AFDEB23

std.json: expose innerParse and add .allocate option (#16312)

3 files changed, 145 insertions(+), 19 deletions(-)

lib
- std
  - json.zig
  - json
    static.zig
    static_test.zig

lib/std/json.zig+1

...	@@ -88,6 +88,7 @@ pub const parseFromSlice = @import("json/static.zig").parseFromSlice;	...	@@ -88,6 +88,7 @@ pub const parseFromSlice = @import("json/static.zig").parseFromSlice;
88	pub const parseFromSliceLeaky = @import("json/static.zig").parseFromSliceLeaky;	88	pub const parseFromSliceLeaky = @import("json/static.zig").parseFromSliceLeaky;
89	pub const parseFromTokenSource = @import("json/static.zig").parseFromTokenSource;	89	pub const parseFromTokenSource = @import("json/static.zig").parseFromTokenSource;
90	pub const parseFromTokenSourceLeaky = @import("json/static.zig").parseFromTokenSourceLeaky;	90	pub const parseFromTokenSourceLeaky = @import("json/static.zig").parseFromTokenSourceLeaky;
		91	pub const innerParse = @import("json/static.zig").innerParse;
91	pub const parseFromValue = @import("json/static.zig").parseFromValue;	92	pub const parseFromValue = @import("json/static.zig").parseFromValue;
92	pub const parseFromValueLeaky = @import("json/static.zig").parseFromValueLeaky;	93	pub const parseFromValueLeaky = @import("json/static.zig").parseFromValueLeaky;
93	pub const ParseError = @import("json/static.zig").ParseError;	94	pub const ParseError = @import("json/static.zig").ParseError;

lib/std/json/static.zig+44-19

 @@ -34,6 +34,14 @@ pub const ParseOptions = struct {
     /// The default for `parseFromTokenSource` with a `*std.json.Reader` is `std.json.default_max_value_len`.
     /// Ignored for `parseFromValue` and `parseFromValueLeaky`.
     max_value_len: ?usize = null,
+    /// This determines whether strings should always be copied,
+    /// or if a reference to the given buffer should be preferred if possible.
+    /// The default for `parseFromSlice` or `parseFromTokenSource` with a `*std.json.Scanner` input
+    /// is `.alloc_if_needed`.
+    /// The default with a `*std.json.Reader` input is `.alloc_always`.
+    /// Ignored for `parseFromValue` and `parseFromValueLeaky`.
+    allocate: ?AllocWhen = null,
 };
 pub fn Parsed(comptime T: type) type {
 @@ -113,7 +121,6 @@ pub fn parseFromTokenSourceLeaky(
     if (@TypeOf(scanner_or_reader.*) == Scanner) {
         assert(scanner_or_reader.is_end_of_input);
     }
     var resolved_options = options;
     if (resolved_options.max_value_len == null) {
         if (@TypeOf(scanner_or_reader.*) == Scanner) {
 @@ -122,8 +129,15 @@ pub fn parseFromTokenSourceLeaky(
             resolved_options.max_value_len = default_max_value_len;
         }
     }
+    if (resolved_options.allocate == null) {
+        if (@TypeOf(scanner_or_reader.*) == Scanner) {
+            resolved_options.allocate = .alloc_if_needed;
+        } else {
+            resolved_options.allocate = .alloc_always;
+        }
+    }
-    const value = try internalParse(T, allocator, scanner_or_reader, resolved_options);
+    const value = try innerParse(T, allocator, scanner_or_reader, resolved_options);
     assert(.end_of_document == try scanner_or_reader.next());
 @@ -181,7 +195,14 @@ pub const ParseFromValueError = std.fmt.ParseIntError || std.fmt.ParseFloatError
     LengthMismatch,
 };
-fn internalParse(
+/// This is an internal function called recursively
+/// during the implementation of `parseFromTokenSourceLeaky` and similar.
+/// It is exposed primarily to enable custom `jsonParse()` methods to call back into the `parseFrom*` system,
+/// such as if you're implementing a custom container of type `T`;
+/// you can call `internalParse(T, ...)` for each of the container's items.
+/// Note that `null` fields are not allowed on the `options` when calling this function.
+/// (The `options` you get in your `jsonParse` method has no `null` fields.)
+pub fn innerParse(
     comptime T: type,
     allocator: Allocator,
     source: anytype,
 @@ -220,7 +241,7 @@ fn internalParse(
                     return null;
                 },
                 else => {
-                    return try internalParse(optionalInfo.child, allocator, source, options);
+                    return try innerParse(optionalInfo.child, allocator, source, options);
                 },
             }
         },
 @@ -250,16 +271,17 @@ fn internalParse(
             var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
             const field_name = switch (name_token.?) {
                 inline .string, .allocated_string => |slice| slice,
-                else => return error.UnexpectedToken,
+                else => {
+                    return error.UnexpectedToken;
+                },
             };
             inline for (unionInfo.fields) |u_field| {
                 if (std.mem.eql(u8, u_field.name, field_name)) {
                     // Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
-                    // (Recursing into internalParse() might trigger more allocations.)
+                    // (Recursing into innerParse() might trigger more allocations.)
                     freeAllocated(allocator, name_token.?);
                     name_token = null;
                     if (u_field.type == void) {
                         // void isn't really a json type, but we can support void payload union tags with {} as a value.
                         if (.object_begin != try source.next()) return error.UnexpectedToken;
 @@ -267,7 +289,7 @@ fn internalParse(
                         result = @unionInit(T, u_field.name, {});
                     } else {
                         // Recurse.
-                        result = @unionInit(T, u_field.name, try internalParse(u_field.type, allocator, source, options));
+                        result = @unionInit(T, u_field.name, try innerParse(u_field.type, allocator, source, options));
                     }
                     break;
                 }
 @@ -287,7 +309,7 @@ fn internalParse(
                 var r: T = undefined;
                 inline for (0..structInfo.fields.len) |i| {
-                    r[i] = try internalParse(structInfo.fields[i].type, allocator, source, options);
+                    r[i] = try innerParse(structInfo.fields[i].type, allocator, source, options);
                 }
                 if (.array_end != try source.next()) return error.UnexpectedToken;
 @@ -307,32 +329,35 @@ fn internalParse(
             while (true) {
                 var name_token: ?Token = try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?);
                 const field_name = switch (name_token.?) {
-                    .object_end => break, // No more fields.
                     inline .string, .allocated_string => |slice| slice,
-                    else => return error.UnexpectedToken,
+                    .object_end => { // No more fields.
+                        break;
+                    },
+                    else => {
+                        return error.UnexpectedToken;
+                    },
                 };
                 inline for (structInfo.fields, 0..) |field, i| {
                     if (field.is_comptime) @compileError("comptime fields are not supported: " ++ @typeName(T) ++ "." ++ field.name);
                     if (std.mem.eql(u8, field.name, field_name)) {
                         // Free the name token now in case we're using an allocator that optimizes freeing the last allocated object.
-                        // (Recursing into internalParse() might trigger more allocations.)
+                        // (Recursing into innerParse() might trigger more allocations.)
                         freeAllocated(allocator, name_token.?);
                         name_token = null;
                         if (fields_seen[i]) {
                             switch (options.duplicate_field_behavior) {
                                 .use_first => {
                                     // Parse and ignore the redundant value.
                                     // We don't want to skip the value, because we want type checking.
-                                    _ = try internalParse(field.type, allocator, source, options);
+                                    _ = try innerParse(field.type, allocator, source, options);
                                     break;
                                 },
                                 .@"error" => return error.DuplicateField,
                                 .use_last => {},
                             }
                         }
-                        @field(r, field.name) = try internalParse(field.type, allocator, source, options);
+                        @field(r, field.name) = try innerParse(field.type, allocator, source, options);
                         fields_seen[i] = true;
                         break;
                     }
 @@ -418,7 +443,7 @@ fn internalParse(
             switch (ptrInfo.size) {
                 .One => {
                     const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
-                    r.* = try internalParse(ptrInfo.child, allocator, source, options);
+                    r.* = try innerParse(ptrInfo.child, allocator, source, options);
                     return r;
                 },
                 .Slice => {
 @@ -438,7 +463,7 @@ fn internalParse(
                                 }
                                 try arraylist.ensureUnusedCapacity(1);
-                                arraylist.appendAssumeCapacity(try internalParse(ptrInfo.child, allocator, source, options));
+                                arraylist.appendAssumeCapacity(try innerParse(ptrInfo.child, allocator, source, options));
                             }
                             if (ptrInfo.sentinel) |some| {
 @@ -459,7 +484,7 @@ fn internalParse(
                                 return try value_list.toOwnedSliceSentinel(@as(*const u8, @ptrCast(sentinel_ptr)).*);
                             }
                             if (ptrInfo.is_const) {
-                                switch (try source.nextAllocMax(allocator, .alloc_if_needed, options.max_value_len.?)) {
+                                switch (try source.nextAllocMax(allocator, options.allocate.?, options.max_value_len.?)) {
                                     inline .string, .allocated_string => |slice| return slice,
                                     else => unreachable,
                                 }
 @@ -495,7 +520,7 @@ fn internalParseArray(
     var r: T = undefined;
     var i: usize = 0;
     while (i < len) : (i += 1) {
-        r[i] = try internalParse(Child, allocator, source, options);
+        r[i] = try innerParse(Child, allocator, source, options);
     }
     if (.array_end != try source.next()) return error.UnexpectedToken;

lib/std/json/static_test.zig+100

...	@@ -7,6 +7,7 @@ const parseFromSlice = @import("./static.zig").parseFromSlice;	...	@@ -7,6 +7,7 @@ const parseFromSlice = @import("./static.zig").parseFromSlice;
7	const parseFromSliceLeaky = @import("./static.zig").parseFromSliceLeaky;	7	const parseFromSliceLeaky = @import("./static.zig").parseFromSliceLeaky;
8	const parseFromTokenSource = @import("./static.zig").parseFromTokenSource;	8	const parseFromTokenSource = @import("./static.zig").parseFromTokenSource;
9	const parseFromTokenSourceLeaky = @import("./static.zig").parseFromTokenSourceLeaky;	9	const parseFromTokenSourceLeaky = @import("./static.zig").parseFromTokenSourceLeaky;
		10	const innerParse = @import("./static.zig").innerParse;
10	const parseFromValue = @import("./static.zig").parseFromValue;	11	const parseFromValue = @import("./static.zig").parseFromValue;
11	const parseFromValueLeaky = @import("./static.zig").parseFromValueLeaky;	12	const parseFromValueLeaky = @import("./static.zig").parseFromValueLeaky;
12	const ParseOptions = @import("./static.zig").ParseOptions;	13	const ParseOptions = @import("./static.zig").ParseOptions;
...	@@ -801,3 +802,102 @@ test "parse into vector" {	...	@@ -801,3 +802,102 @@ test "parse into vector" {
801	try testing.expectApproxEqAbs(@as(f32, 2.5), parsed.value.vec_f32[1], 0.0000001);	802	try testing.expectApproxEqAbs(@as(f32, 2.5), parsed.value.vec_f32[1], 0.0000001);
802	try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, parsed.value.vec_i32);	803	try testing.expectEqual(@Vector(4, i32){ 4, 5, 6, 7 }, parsed.value.vec_i32);
803	}	804	}
		805
		806	fn assertKey(
		807	allocator: Allocator,
		808	test_string: []const u8,
		809	scanner: anytype,
		810	) !void {
		811	const token_outer = try scanner.nextAlloc(allocator, .alloc_always);
		812	switch (token_outer) {
		813	.allocated_string => \|string\| {
		814	try testing.expectEqualSlices(u8, string, test_string);
		815	allocator.free(string);
		816	},
		817	else => return error.UnexpectedToken,
		818	}
		819	}
		820	test "json parse partial" {
		821	const Inner = struct {
		822	num: u32,
		823	yes: bool,
		824	};
		825	var str =
		826	\\{
		827	\\ "outer": {
		828	\\ "key1": {
		829	\\ "num": 75,
		830	\\ "yes": true
		831	\\ },
		832	\\ "key2": {
		833	\\ "num": 95,
		834	\\ "yes": false
		835	\\ }
		836	\\ }
		837	\\}
		838	;
		839	var allocator = testing.allocator;
		840	var scanner = JsonScanner.initCompleteInput(allocator, str);
		841	defer scanner.deinit();
		842
		843	var arena = ArenaAllocator.init(allocator);
		844	defer arena.deinit();
		845
		846	// Peel off the outer object
		847	try testing.expectEqual(try scanner.next(), .object_begin);
		848	try assertKey(allocator, "outer", &scanner);
		849	try testing.expectEqual(try scanner.next(), .object_begin);
		850	try assertKey(allocator, "key1", &scanner);
		851
		852	// Parse the inner object to an Inner struct
		853	const inner_token = try innerParse(
		854	Inner,
		855	arena.allocator(),
		856	&scanner,
		857	.{ .max_value_len = scanner.input.len },
		858	);
		859	try testing.expectEqual(inner_token.num, 75);
		860	try testing.expectEqual(inner_token.yes, true);
		861
		862	// Get they next key
		863	try assertKey(allocator, "key2", &scanner);
		864	const inner_token_2 = try innerParse(
		865	Inner,
		866	arena.allocator(),
		867	&scanner,
		868	.{ .max_value_len = scanner.input.len },
		869	);
		870	try testing.expectEqual(inner_token_2.num, 95);
		871	try testing.expectEqual(inner_token_2.yes, false);
		872	try testing.expectEqual(try scanner.next(), .object_end);
		873	}
		874
		875	test "json parse allocate when streaming" {
		876	const T = struct {
		877	not_const: []u8,
		878	is_const: []const u8,
		879	};
		880	var str =
		881	\\{
		882	\\ "not_const": "non const string",
		883	\\ "is_const": "const string"
		884	\\}
		885	;
		886	var allocator = testing.allocator;
		887	var arena = ArenaAllocator.init(allocator);
		888	defer arena.deinit();
		889
		890	var stream = std.io.fixedBufferStream(str);
		891	var json_reader = jsonReader(std.testing.allocator, stream.reader());
		892
		893	const parsed = parseFromTokenSourceLeaky(T, arena.allocator(), &json_reader, .{}) catch \|err\| {
		894	json_reader.deinit();
		895	return err;
		896	};
		897	// Deinit our reader to invalidate its buffer
		898	json_reader.deinit();
		899
		900	// If either of these was invalidated, it would be full of '0xAA'
		901	try testing.expectEqualSlices(u8, parsed.not_const, "non const string");
		902	try testing.expectEqualSlices(u8, parsed.is_const, "const string");
		903	}