authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2023-07-09 22:18:59-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-07-09 22:18:59-04:00
log874d2dd9f77b60a7eb6b2af3c34bb02783b0ec85
treec54a549b684fd925c1986d268e9458b3fb539159
parenta7553107345bab99b0f5318e0fd3efae84f56b56
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.json: add generic hash map that parses/stringifies with arbitrary string keys (#16366)

* expose innerParseFromValue

4 files changed, 266 insertions(+), 25 deletions(-)

lib/std/json.zig+4
...@@ -69,6 +69,8 @@ pub const ObjectMap = @import("json/dynamic.zig").ObjectMap;...@@ -69,6 +69,8 @@ pub const ObjectMap = @import("json/dynamic.zig").ObjectMap;
69pub const Array = @import("json/dynamic.zig").Array;69pub const Array = @import("json/dynamic.zig").Array;
70pub const Value = @import("json/dynamic.zig").Value;70pub const Value = @import("json/dynamic.zig").Value;
7171
72pub const ArrayHashMap = @import("json/hashmap.zig").ArrayHashMap;
73
72pub const validate = @import("json/scanner.zig").validate;74pub const validate = @import("json/scanner.zig").validate;
73pub const Error = @import("json/scanner.zig").Error;75pub const Error = @import("json/scanner.zig").Error;
74pub const reader = @import("json/scanner.zig").reader;76pub const reader = @import("json/scanner.zig").reader;
...@@ -91,6 +93,7 @@ pub const parseFromTokenSourceLeaky = @import("json/static.zig").parseFromTokenS...@@ -91,6 +93,7 @@ pub const parseFromTokenSourceLeaky = @import("json/static.zig").parseFromTokenS
91pub const innerParse = @import("json/static.zig").innerParse;93pub const innerParse = @import("json/static.zig").innerParse;
92pub const parseFromValue = @import("json/static.zig").parseFromValue;94pub const parseFromValue = @import("json/static.zig").parseFromValue;
93pub const parseFromValueLeaky = @import("json/static.zig").parseFromValueLeaky;95pub const parseFromValueLeaky = @import("json/static.zig").parseFromValueLeaky;
96pub const innerParseFromValue = @import("json/static.zig").innerParseFromValue;
94pub const ParseError = @import("json/static.zig").ParseError;97pub const ParseError = @import("json/static.zig").ParseError;
95pub const ParseFromValueError = @import("json/static.zig").ParseFromValueError;98pub const ParseFromValueError = @import("json/static.zig").ParseFromValueError;
9699
...@@ -116,6 +119,7 @@ test {...@@ -116,6 +119,7 @@ test {
116 _ = @import("json/scanner.zig");119 _ = @import("json/scanner.zig");
117 _ = @import("json/write_stream.zig");120 _ = @import("json/write_stream.zig");
118 _ = @import("json/dynamic.zig");121 _ = @import("json/dynamic.zig");
122 _ = @import("json/hashmap_test.zig");
119 _ = @import("json/static.zig");123 _ = @import("json/static.zig");
120 _ = @import("json/stringify.zig");124 _ = @import("json/stringify.zig");
121 _ = @import("json/JSONTestSuite_test.zig");125 _ = @import("json/JSONTestSuite_test.zig");
lib/std/json/hashmap.zig created+103
...@@ -0,0 +1,103 @@
1const std = @import("std");
2const Allocator = std.mem.Allocator;
3
4const ParseOptions = @import("static.zig").ParseOptions;
5const innerParse = @import("static.zig").innerParse;
6const innerParseFromValue = @import("static.zig").innerParseFromValue;
7const Value = @import("dynamic.zig").Value;
8const StringifyOptions = @import("stringify.zig").StringifyOptions;
9const stringify = @import("stringify.zig").stringify;
10const encodeJsonString = @import("stringify.zig").encodeJsonString;
11
12/// A thin wrapper around `std.StringArrayHashMapUnmanaged` that implements
13/// `jsonParse`, `jsonParseFromValue`, and `jsonStringify`.
14/// This is useful when your JSON schema has an object with arbitrary data keys
15/// instead of comptime-known struct field names.
16pub fn ArrayHashMap(comptime T: type) type {
17 return struct {
18 map: std.StringArrayHashMapUnmanaged(T) = .{},
19
20 pub fn deinit(self: *@This(), allocator: Allocator) void {
21 self.map.deinit(allocator);
22 }
23
24 pub fn jsonParse(allocator: Allocator, source: anytype, options: ParseOptions) !@This() {
25 var map = std.StringArrayHashMapUnmanaged(T){};
26 errdefer map.deinit(allocator);
27
28 if (.object_begin != try source.next()) return error.UnexpectedToken;
29 while (true) {
30 const token = try source.nextAlloc(allocator, .alloc_if_needed);
31 switch (token) {
32 inline .string, .allocated_string => |k| {
33 const gop = try map.getOrPut(allocator, k);
34 if (token == .allocated_string) {
35 // Free the key before recursing in case we're using an allocator
36 // that optimizes freeing the last allocated object.
37 allocator.free(k);
38 }
39 if (gop.found_existing) {
40 switch (options.duplicate_field_behavior) {
41 .use_first => {
42 // Parse and ignore the redundant value.
43 // We don't want to skip the value, because we want type checking.
44 _ = try innerParse(T, allocator, source, options);
45 continue;
46 },
47 .@"error" => return error.DuplicateField,
48 .use_last => {},
49 }
50 }
51 gop.value_ptr.* = try innerParse(T, allocator, source, options);
52 },
53 .object_end => break,
54 else => unreachable,
55 }
56 }
57 return .{ .map = map };
58 }
59
60 pub fn jsonParseFromValue(allocator: Allocator, source: Value, options: ParseOptions) !@This() {
61 if (source != .object) return error.UnexpectedToken;
62
63 var map = std.StringArrayHashMapUnmanaged(T){};
64 errdefer map.deinit(allocator);
65
66 var it = source.object.iterator();
67 while (it.next()) |kv| {
68 try map.put(allocator, kv.key_ptr.*, try innerParseFromValue(T, allocator, kv.value_ptr.*, options));
69 }
70 return .{ .map = map };
71 }
72
73 pub fn jsonStringify(self: @This(), options: StringifyOptions, out_stream: anytype) !void {
74 try out_stream.writeByte('{');
75 var field_output = false;
76 var child_options = options;
77 child_options.whitespace.indent_level += 1;
78 var it = self.map.iterator();
79 while (it.next()) |kv| {
80 if (!field_output) {
81 field_output = true;
82 } else {
83 try out_stream.writeByte(',');
84 }
85 try child_options.whitespace.outputIndent(out_stream);
86 try encodeJsonString(kv.key_ptr.*, options, out_stream);
87 try out_stream.writeByte(':');
88 if (child_options.whitespace.separator) {
89 try out_stream.writeByte(' ');
90 }
91 try stringify(kv.value_ptr.*, child_options, out_stream);
92 }
93 if (field_output) {
94 try options.whitespace.outputIndent(out_stream);
95 }
96 try out_stream.writeByte('}');
97 }
98 };
99}
100
101test {
102 _ = @import("hashmap_test.zig");
103}
lib/std/json/hashmap_test.zig created+139
...@@ -0,0 +1,139 @@
1const std = @import("std");
2const testing = std.testing;
3
4const ArrayHashMap = @import("hashmap.zig").ArrayHashMap;
5
6const parseFromSlice = @import("static.zig").parseFromSlice;
7const parseFromSliceLeaky = @import("static.zig").parseFromSliceLeaky;
8const parseFromValue = @import("static.zig").parseFromValue;
9const stringifyAlloc = @import("stringify.zig").stringifyAlloc;
10const Value = @import("dynamic.zig").Value;
11
12const T = struct {
13 i: i32,
14 s: []const u8,
15};
16
17test "parse json hashmap" {
18 const doc =
19 \\{
20 \\ "abc": {"i": 0, "s": "d"},
21 \\ "xyz": {"i": 1, "s": "w"}
22 \\}
23 ;
24 const parsed = try parseFromSlice(ArrayHashMap(T), testing.allocator, doc, .{});
25 defer parsed.deinit();
26
27 try testing.expectEqual(@as(usize, 2), parsed.value.map.count());
28 try testing.expectEqualStrings("d", parsed.value.map.get("abc").?.s);
29 try testing.expectEqual(@as(i32, 1), parsed.value.map.get("xyz").?.i);
30}
31
32test "parse json hashmap duplicate fields" {
33 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
34 defer arena.deinit();
35
36 const doc =
37 \\{
38 \\ "abc": {"i": 0, "s": "d"},
39 \\ "abc": {"i": 1, "s": "w"}
40 \\}
41 ;
42
43 try testing.expectError(error.DuplicateField, parseFromSliceLeaky(ArrayHashMap(T), arena.allocator(), doc, .{
44 .duplicate_field_behavior = .@"error",
45 }));
46
47 const first = try parseFromSliceLeaky(ArrayHashMap(T), arena.allocator(), doc, .{
48 .duplicate_field_behavior = .use_first,
49 });
50 try testing.expectEqual(@as(usize, 1), first.map.count());
51 try testing.expectEqual(@as(i32, 0), first.map.get("abc").?.i);
52
53 const last = try parseFromSliceLeaky(ArrayHashMap(T), arena.allocator(), doc, .{
54 .duplicate_field_behavior = .use_last,
55 });
56 try testing.expectEqual(@as(usize, 1), last.map.count());
57 try testing.expectEqual(@as(i32, 1), last.map.get("abc").?.i);
58}
59
60test "stringify json hashmap" {
61 var value = ArrayHashMap(T){};
62 defer value.deinit(testing.allocator);
63 {
64 const doc = try stringifyAlloc(testing.allocator, value, .{});
65 defer testing.allocator.free(doc);
66 try testing.expectEqualStrings("{}", doc);
67 }
68
69 try value.map.put(testing.allocator, "abc", .{ .i = 0, .s = "d" });
70 try value.map.put(testing.allocator, "xyz", .{ .i = 1, .s = "w" });
71
72 {
73 const doc = try stringifyAlloc(testing.allocator, value, .{});
74 defer testing.allocator.free(doc);
75 try testing.expectEqualStrings(
76 \\{"abc":{"i":0,"s":"d"},"xyz":{"i":1,"s":"w"}}
77 , doc);
78 }
79
80 try testing.expect(value.map.swapRemove("abc"));
81 {
82 const doc = try stringifyAlloc(testing.allocator, value, .{});
83 defer testing.allocator.free(doc);
84 try testing.expectEqualStrings(
85 \\{"xyz":{"i":1,"s":"w"}}
86 , doc);
87 }
88
89 try testing.expect(value.map.swapRemove("xyz"));
90 {
91 const doc = try stringifyAlloc(testing.allocator, value, .{});
92 defer testing.allocator.free(doc);
93 try testing.expectEqualStrings("{}", doc);
94 }
95}
96
97test "stringify json hashmap whitespace" {
98 var value = ArrayHashMap(T){};
99 defer value.deinit(testing.allocator);
100 try value.map.put(testing.allocator, "abc", .{ .i = 0, .s = "d" });
101 try value.map.put(testing.allocator, "xyz", .{ .i = 1, .s = "w" });
102
103 {
104 const doc = try stringifyAlloc(testing.allocator, value, .{
105 .whitespace = .{
106 .indent = .{ .space = 2 },
107 },
108 });
109 defer testing.allocator.free(doc);
110 try testing.expectEqualStrings(
111 \\{
112 \\ "abc": {
113 \\ "i": 0,
114 \\ "s": "d"
115 \\ },
116 \\ "xyz": {
117 \\ "i": 1,
118 \\ "s": "w"
119 \\ }
120 \\}
121 , doc);
122 }
123}
124
125test "json parse from value hashmap" {
126 const doc =
127 \\{
128 \\ "abc": {"i": 0, "s": "d"},
129 \\ "xyz": {"i": 1, "s": "w"}
130 \\}
131 ;
132 const parsed1 = try parseFromSlice(Value, testing.allocator, doc, .{});
133 defer parsed1.deinit();
134
135 const parsed2 = try parseFromValue(ArrayHashMap(T), testing.allocator, parsed1.value, .{});
136 defer parsed2.deinit();
137
138 try testing.expectEqualStrings("d", parsed2.value.map.get("abc").?.s);
139}
lib/std/json/static.zig+20-25
...@@ -145,6 +145,7 @@ pub fn parseFromTokenSourceLeaky(...@@ -145,6 +145,7 @@ pub fn parseFromTokenSourceLeaky(
145}145}
146146
147/// Like `parseFromSlice`, but the input is an already-parsed `std.json.Value` object.147/// Like `parseFromSlice`, but the input is an already-parsed `std.json.Value` object.
148/// Only `options.ignore_unknown_fields` is used from `options`.
148pub fn parseFromValue(149pub fn parseFromValue(
149 comptime T: type,150 comptime T: type,
150 allocator: Allocator,151 allocator: Allocator,
...@@ -173,7 +174,7 @@ pub fn parseFromValueLeaky(...@@ -173,7 +174,7 @@ pub fn parseFromValueLeaky(
173 // I guess this function doesn't need to exist,174 // I guess this function doesn't need to exist,
174 // but the flow of the sourcecode is easy to follow and grouped nicely with175 // but the flow of the sourcecode is easy to follow and grouped nicely with
175 // this pub redirect function near the top and the implementation near the bottom.176 // this pub redirect function near the top and the implementation near the bottom.
176 return internalParseFromValue(T, allocator, source, options);177 return innerParseFromValue(T, allocator, source, options);
177}178}
178179
179/// The error set that will be returned when parsing from `*Source`.180/// The error set that will be returned when parsing from `*Source`.
...@@ -199,7 +200,7 @@ pub const ParseFromValueError = std.fmt.ParseIntError || std.fmt.ParseFloatError...@@ -199,7 +200,7 @@ pub const ParseFromValueError = std.fmt.ParseIntError || std.fmt.ParseFloatError
199/// during the implementation of `parseFromTokenSourceLeaky` and similar.200/// during the implementation of `parseFromTokenSourceLeaky` and similar.
200/// It is exposed primarily to enable custom `jsonParse()` methods to call back into the `parseFrom*` system,201/// It is exposed primarily to enable custom `jsonParse()` methods to call back into the `parseFrom*` system,
201/// such as if you're implementing a custom container of type `T`;202/// such as if you're implementing a custom container of type `T`;
202/// you can call `internalParse(T, ...)` for each of the container's items.203/// you can call `innerParse(T, ...)` for each of the container's items.
203/// Note that `null` fields are not allowed on the `options` when calling this function.204/// Note that `null` fields are not allowed on the `options` when calling this function.
204/// (The `options` you get in your `jsonParse` method has no `null` fields.)205/// (The `options` you get in your `jsonParse` method has no `null` fields.)
205pub fn innerParse(206pub fn innerParse(
...@@ -528,7 +529,12 @@ fn internalParseArray(...@@ -528,7 +529,12 @@ fn internalParseArray(
528 return r;529 return r;
529}530}
530531
531fn internalParseFromValue(532/// This is an internal function called recursively
533/// during the implementation of `parseFromValueLeaky`.
534/// It is exposed primarily to enable custom `jsonParseFromValue()` methods to call back into the `parseFromValue*` system,
535/// such as if you're implementing a custom container of type `T`;
536/// you can call `innerParseFromValue(T, ...)` for each of the container's items.
537pub fn innerParseFromValue(
532 comptime T: type,538 comptime T: type,
533 allocator: Allocator,539 allocator: Allocator,
534 source: Value,540 source: Value,
...@@ -571,7 +577,7 @@ fn internalParseFromValue(...@@ -571,7 +577,7 @@ fn internalParseFromValue(
571 .Optional => |optionalInfo| {577 .Optional => |optionalInfo| {
572 switch (source) {578 switch (source) {
573 .null => return null,579 .null => return null,
574 else => return try internalParseFromValue(optionalInfo.child, allocator, source, options),580 else => return try innerParseFromValue(optionalInfo.child, allocator, source, options),
575 }581 }
576 },582 },
577 .Enum => {583 .Enum => {
...@@ -609,7 +615,7 @@ fn internalParseFromValue(...@@ -609,7 +615,7 @@ fn internalParseFromValue(
609 return @unionInit(T, u_field.name, {});615 return @unionInit(T, u_field.name, {});
610 }616 }
611 // Recurse.617 // Recurse.
612 return @unionInit(T, u_field.name, try internalParseFromValue(u_field.type, allocator, kv.value_ptr.*, options));618 return @unionInit(T, u_field.name, try innerParseFromValue(u_field.type, allocator, kv.value_ptr.*, options));
613 }619 }
614 }620 }
615 // Didn't match anything.621 // Didn't match anything.
...@@ -623,7 +629,7 @@ fn internalParseFromValue(...@@ -623,7 +629,7 @@ fn internalParseFromValue(
623629
624 var r: T = undefined;630 var r: T = undefined;
625 inline for (0..structInfo.fields.len, source.array.items) |i, item| {631 inline for (0..structInfo.fields.len, source.array.items) |i, item| {
626 r[i] = try internalParseFromValue(structInfo.fields[i].type, allocator, item, options);632 r[i] = try innerParseFromValue(structInfo.fields[i].type, allocator, item, options);
627 }633 }
628634
629 return r;635 return r;
...@@ -645,19 +651,8 @@ fn internalParseFromValue(...@@ -645,19 +651,8 @@ fn internalParseFromValue(
645 inline for (structInfo.fields, 0..) |field, i| {651 inline for (structInfo.fields, 0..) |field, i| {
646 if (field.is_comptime) @compileError("comptime fields are not supported: " ++ @typeName(T) ++ "." ++ field.name);652 if (field.is_comptime) @compileError("comptime fields are not supported: " ++ @typeName(T) ++ "." ++ field.name);
647 if (std.mem.eql(u8, field.name, field_name)) {653 if (std.mem.eql(u8, field.name, field_name)) {
648 if (fields_seen[i]) {654 assert(!fields_seen[i]); // Can't have duplicate keys in a Value.object.
649 switch (options.duplicate_field_behavior) {655 @field(r, field.name) = try innerParseFromValue(field.type, allocator, kv.value_ptr.*, options);
650 .use_first => {
651 // Parse and ignore the redundant value.
652 // We don't want to skip the value, because we want type checking.
653 _ = try internalParseFromValue(field.type, allocator, kv.value_ptr.*, options);
654 break;
655 },
656 .@"error" => return error.DuplicateField,
657 .use_last => {},
658 }
659 }
660 @field(r, field.name) = try internalParseFromValue(field.type, allocator, kv.value_ptr.*, options);
661 fields_seen[i] = true;656 fields_seen[i] = true;
662 break;657 break;
663 }658 }
...@@ -674,7 +669,7 @@ fn internalParseFromValue(...@@ -674,7 +669,7 @@ fn internalParseFromValue(
674 switch (source) {669 switch (source) {
675 .array => |array| {670 .array => |array| {
676 // Typical array.671 // Typical array.
677 return internalParseArrayFromArrayValue(T, arrayInfo.child, arrayInfo.len, allocator, array, options);672 return innerParseArrayFromArrayValue(T, arrayInfo.child, arrayInfo.len, allocator, array, options);
678 },673 },
679 .string => |s| {674 .string => |s| {
680 if (arrayInfo.child != u8) return error.UnexpectedToken;675 if (arrayInfo.child != u8) return error.UnexpectedToken;
...@@ -694,7 +689,7 @@ fn internalParseFromValue(...@@ -694,7 +689,7 @@ fn internalParseFromValue(
694 .Vector => |vecInfo| {689 .Vector => |vecInfo| {
695 switch (source) {690 switch (source) {
696 .array => |array| {691 .array => |array| {
697 return internalParseArrayFromArrayValue(T, vecInfo.child, vecInfo.len, allocator, array, options);692 return innerParseArrayFromArrayValue(T, vecInfo.child, vecInfo.len, allocator, array, options);
698 },693 },
699 else => return error.UnexpectedToken,694 else => return error.UnexpectedToken,
700 }695 }
...@@ -704,7 +699,7 @@ fn internalParseFromValue(...@@ -704,7 +699,7 @@ fn internalParseFromValue(
704 switch (ptrInfo.size) {699 switch (ptrInfo.size) {
705 .One => {700 .One => {
706 const r: *ptrInfo.child = try allocator.create(ptrInfo.child);701 const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
707 r.* = try internalParseFromValue(ptrInfo.child, allocator, source, options);702 r.* = try innerParseFromValue(ptrInfo.child, allocator, source, options);
708 return r;703 return r;
709 },704 },
710 .Slice => {705 .Slice => {
...@@ -716,7 +711,7 @@ fn internalParseFromValue(...@@ -716,7 +711,7 @@ fn internalParseFromValue(
716 try allocator.alloc(ptrInfo.child, array.items.len);711 try allocator.alloc(ptrInfo.child, array.items.len);
717712
718 for (array.items, r) |item, *dest| {713 for (array.items, r) |item, *dest| {
719 dest.* = try internalParseFromValue(ptrInfo.child, allocator, item, options);714 dest.* = try innerParseFromValue(ptrInfo.child, allocator, item, options);
720 }715 }
721716
722 return r;717 return r;
...@@ -743,7 +738,7 @@ fn internalParseFromValue(...@@ -743,7 +738,7 @@ fn internalParseFromValue(
743 }738 }
744}739}
745740
746fn internalParseArrayFromArrayValue(741fn innerParseArrayFromArrayValue(
747 comptime T: type,742 comptime T: type,
748 comptime Child: type,743 comptime Child: type,
749 comptime len: comptime_int,744 comptime len: comptime_int,
...@@ -755,7 +750,7 @@ fn internalParseArrayFromArrayValue(...@@ -755,7 +750,7 @@ fn internalParseArrayFromArrayValue(
755750
756 var r: T = undefined;751 var r: T = undefined;
757 for (array.items, 0..) |item, i| {752 for (array.items, 0..) |item, i| {
758 r[i] = try internalParseFromValue(Child, allocator, item, options);753 r[i] = try innerParseFromValue(Child, allocator, item, options);
759 }754 }
760755
761 return r;756 return r;