authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2023-07-09 22:18:59-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-07-09 22:18:59-04:00
log874d2dd9f77b60a7eb6b2af3c34bb02783b0ec85
treec54a549b684fd925c1986d268e9458b3fb539159
parenta7553107345bab99b0f5318e0fd3efae84f56b56
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.json: add generic hash map that parses/stringifies with arbitrary string keys (#16366)

* expose innerParseFromValue

4 files changed, 266 insertions(+), 25 deletions(-)

lib/std/json.zig+4
......@@ -69,6 +69,8 @@ pub const ObjectMap = @import("json/dynamic.zig").ObjectMap;
6969pub const Array = @import("json/dynamic.zig").Array;
7070pub const Value = @import("json/dynamic.zig").Value;
7171
72pub const ArrayHashMap = @import("json/hashmap.zig").ArrayHashMap;
73
7274pub const validate = @import("json/scanner.zig").validate;
7375pub const Error = @import("json/scanner.zig").Error;
7476pub const reader = @import("json/scanner.zig").reader;
......@@ -91,6 +93,7 @@ pub const parseFromTokenSourceLeaky = @import("json/static.zig").parseFromTokenS
9193pub const innerParse = @import("json/static.zig").innerParse;
9294pub const parseFromValue = @import("json/static.zig").parseFromValue;
9395pub const parseFromValueLeaky = @import("json/static.zig").parseFromValueLeaky;
96pub const innerParseFromValue = @import("json/static.zig").innerParseFromValue;
9497pub const ParseError = @import("json/static.zig").ParseError;
9598pub const ParseFromValueError = @import("json/static.zig").ParseFromValueError;
9699
......@@ -116,6 +119,7 @@ test {
116119 _ = @import("json/scanner.zig");
117120 _ = @import("json/write_stream.zig");
118121 _ = @import("json/dynamic.zig");
122 _ = @import("json/hashmap_test.zig");
119123 _ = @import("json/static.zig");
120124 _ = @import("json/stringify.zig");
121125 _ = @import("json/JSONTestSuite_test.zig");
lib/std/json/hashmap.zig created+103
......@@ -0,0 +1,103 @@
1const std = @import("std");
2const Allocator = std.mem.Allocator;
3
4const ParseOptions = @import("static.zig").ParseOptions;
5const innerParse = @import("static.zig").innerParse;
6const innerParseFromValue = @import("static.zig").innerParseFromValue;
7const Value = @import("dynamic.zig").Value;
8const StringifyOptions = @import("stringify.zig").StringifyOptions;
9const stringify = @import("stringify.zig").stringify;
10const encodeJsonString = @import("stringify.zig").encodeJsonString;
11
12/// A thin wrapper around `std.StringArrayHashMapUnmanaged` that implements
13/// `jsonParse`, `jsonParseFromValue`, and `jsonStringify`.
14/// This is useful when your JSON schema has an object with arbitrary data keys
15/// instead of comptime-known struct field names.
16pub fn ArrayHashMap(comptime T: type) type {
17 return struct {
18 map: std.StringArrayHashMapUnmanaged(T) = .{},
19
20 pub fn deinit(self: *@This(), allocator: Allocator) void {
21 self.map.deinit(allocator);
22 }
23
24 pub fn jsonParse(allocator: Allocator, source: anytype, options: ParseOptions) !@This() {
25 var map = std.StringArrayHashMapUnmanaged(T){};
26 errdefer map.deinit(allocator);
27
28 if (.object_begin != try source.next()) return error.UnexpectedToken;
29 while (true) {
30 const token = try source.nextAlloc(allocator, .alloc_if_needed);
31 switch (token) {
32 inline .string, .allocated_string => |k| {
33 const gop = try map.getOrPut(allocator, k);
34 if (token == .allocated_string) {
35 // Free the key before recursing in case we're using an allocator
36 // that optimizes freeing the last allocated object.
37 allocator.free(k);
38 }
39 if (gop.found_existing) {
40 switch (options.duplicate_field_behavior) {
41 .use_first => {
42 // Parse and ignore the redundant value.
43 // We don't want to skip the value, because we want type checking.
44 _ = try innerParse(T, allocator, source, options);
45 continue;
46 },
47 .@"error" => return error.DuplicateField,
48 .use_last => {},
49 }
50 }
51 gop.value_ptr.* = try innerParse(T, allocator, source, options);
52 },
53 .object_end => break,
54 else => unreachable,
55 }
56 }
57 return .{ .map = map };
58 }
59
60 pub fn jsonParseFromValue(allocator: Allocator, source: Value, options: ParseOptions) !@This() {
61 if (source != .object) return error.UnexpectedToken;
62
63 var map = std.StringArrayHashMapUnmanaged(T){};
64 errdefer map.deinit(allocator);
65
66 var it = source.object.iterator();
67 while (it.next()) |kv| {
68 try map.put(allocator, kv.key_ptr.*, try innerParseFromValue(T, allocator, kv.value_ptr.*, options));
69 }
70 return .{ .map = map };
71 }
72
73 pub fn jsonStringify(self: @This(), options: StringifyOptions, out_stream: anytype) !void {
74 try out_stream.writeByte('{');
75 var field_output = false;
76 var child_options = options;
77 child_options.whitespace.indent_level += 1;
78 var it = self.map.iterator();
79 while (it.next()) |kv| {
80 if (!field_output) {
81 field_output = true;
82 } else {
83 try out_stream.writeByte(',');
84 }
85 try child_options.whitespace.outputIndent(out_stream);
86 try encodeJsonString(kv.key_ptr.*, options, out_stream);
87 try out_stream.writeByte(':');
88 if (child_options.whitespace.separator) {
89 try out_stream.writeByte(' ');
90 }
91 try stringify(kv.value_ptr.*, child_options, out_stream);
92 }
93 if (field_output) {
94 try options.whitespace.outputIndent(out_stream);
95 }
96 try out_stream.writeByte('}');
97 }
98 };
99}
100
101test {
102 _ = @import("hashmap_test.zig");
103}
lib/std/json/hashmap_test.zig created+139
......@@ -0,0 +1,139 @@
1const std = @import("std");
2const testing = std.testing;
3
4const ArrayHashMap = @import("hashmap.zig").ArrayHashMap;
5
6const parseFromSlice = @import("static.zig").parseFromSlice;
7const parseFromSliceLeaky = @import("static.zig").parseFromSliceLeaky;
8const parseFromValue = @import("static.zig").parseFromValue;
9const stringifyAlloc = @import("stringify.zig").stringifyAlloc;
10const Value = @import("dynamic.zig").Value;
11
12const T = struct {
13 i: i32,
14 s: []const u8,
15};
16
17test "parse json hashmap" {
18 const doc =
19 \\{
20 \\ "abc": {"i": 0, "s": "d"},
21 \\ "xyz": {"i": 1, "s": "w"}
22 \\}
23 ;
24 const parsed = try parseFromSlice(ArrayHashMap(T), testing.allocator, doc, .{});
25 defer parsed.deinit();
26
27 try testing.expectEqual(@as(usize, 2), parsed.value.map.count());
28 try testing.expectEqualStrings("d", parsed.value.map.get("abc").?.s);
29 try testing.expectEqual(@as(i32, 1), parsed.value.map.get("xyz").?.i);
30}
31
32test "parse json hashmap duplicate fields" {
33 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
34 defer arena.deinit();
35
36 const doc =
37 \\{
38 \\ "abc": {"i": 0, "s": "d"},
39 \\ "abc": {"i": 1, "s": "w"}
40 \\}
41 ;
42
43 try testing.expectError(error.DuplicateField, parseFromSliceLeaky(ArrayHashMap(T), arena.allocator(), doc, .{
44 .duplicate_field_behavior = .@"error",
45 }));
46
47 const first = try parseFromSliceLeaky(ArrayHashMap(T), arena.allocator(), doc, .{
48 .duplicate_field_behavior = .use_first,
49 });
50 try testing.expectEqual(@as(usize, 1), first.map.count());
51 try testing.expectEqual(@as(i32, 0), first.map.get("abc").?.i);
52
53 const last = try parseFromSliceLeaky(ArrayHashMap(T), arena.allocator(), doc, .{
54 .duplicate_field_behavior = .use_last,
55 });
56 try testing.expectEqual(@as(usize, 1), last.map.count());
57 try testing.expectEqual(@as(i32, 1), last.map.get("abc").?.i);
58}
59
60test "stringify json hashmap" {
61 var value = ArrayHashMap(T){};
62 defer value.deinit(testing.allocator);
63 {
64 const doc = try stringifyAlloc(testing.allocator, value, .{});
65 defer testing.allocator.free(doc);
66 try testing.expectEqualStrings("{}", doc);
67 }
68
69 try value.map.put(testing.allocator, "abc", .{ .i = 0, .s = "d" });
70 try value.map.put(testing.allocator, "xyz", .{ .i = 1, .s = "w" });
71
72 {
73 const doc = try stringifyAlloc(testing.allocator, value, .{});
74 defer testing.allocator.free(doc);
75 try testing.expectEqualStrings(
76 \\{"abc":{"i":0,"s":"d"},"xyz":{"i":1,"s":"w"}}
77 , doc);
78 }
79
80 try testing.expect(value.map.swapRemove("abc"));
81 {
82 const doc = try stringifyAlloc(testing.allocator, value, .{});
83 defer testing.allocator.free(doc);
84 try testing.expectEqualStrings(
85 \\{"xyz":{"i":1,"s":"w"}}
86 , doc);
87 }
88
89 try testing.expect(value.map.swapRemove("xyz"));
90 {
91 const doc = try stringifyAlloc(testing.allocator, value, .{});
92 defer testing.allocator.free(doc);
93 try testing.expectEqualStrings("{}", doc);
94 }
95}
96
97test "stringify json hashmap whitespace" {
98 var value = ArrayHashMap(T){};
99 defer value.deinit(testing.allocator);
100 try value.map.put(testing.allocator, "abc", .{ .i = 0, .s = "d" });
101 try value.map.put(testing.allocator, "xyz", .{ .i = 1, .s = "w" });
102
103 {
104 const doc = try stringifyAlloc(testing.allocator, value, .{
105 .whitespace = .{
106 .indent = .{ .space = 2 },
107 },
108 });
109 defer testing.allocator.free(doc);
110 try testing.expectEqualStrings(
111 \\{
112 \\ "abc": {
113 \\ "i": 0,
114 \\ "s": "d"
115 \\ },
116 \\ "xyz": {
117 \\ "i": 1,
118 \\ "s": "w"
119 \\ }
120 \\}
121 , doc);
122 }
123}
124
125test "json parse from value hashmap" {
126 const doc =
127 \\{
128 \\ "abc": {"i": 0, "s": "d"},
129 \\ "xyz": {"i": 1, "s": "w"}
130 \\}
131 ;
132 const parsed1 = try parseFromSlice(Value, testing.allocator, doc, .{});
133 defer parsed1.deinit();
134
135 const parsed2 = try parseFromValue(ArrayHashMap(T), testing.allocator, parsed1.value, .{});
136 defer parsed2.deinit();
137
138 try testing.expectEqualStrings("d", parsed2.value.map.get("abc").?.s);
139}
lib/std/json/static.zig+20-25
......@@ -145,6 +145,7 @@ pub fn parseFromTokenSourceLeaky(
145145}
146146
147147/// Like `parseFromSlice`, but the input is an already-parsed `std.json.Value` object.
148/// Only `options.ignore_unknown_fields` is used from `options`.
148149pub fn parseFromValue(
149150 comptime T: type,
150151 allocator: Allocator,
......@@ -173,7 +174,7 @@ pub fn parseFromValueLeaky(
173174 // I guess this function doesn't need to exist,
174175 // but the flow of the sourcecode is easy to follow and grouped nicely with
175176 // this pub redirect function near the top and the implementation near the bottom.
176 return internalParseFromValue(T, allocator, source, options);
177 return innerParseFromValue(T, allocator, source, options);
177178}
178179
179180/// The error set that will be returned when parsing from `*Source`.
......@@ -199,7 +200,7 @@ pub const ParseFromValueError = std.fmt.ParseIntError || std.fmt.ParseFloatError
199200/// during the implementation of `parseFromTokenSourceLeaky` and similar.
200201/// It is exposed primarily to enable custom `jsonParse()` methods to call back into the `parseFrom*` system,
201202/// such as if you're implementing a custom container of type `T`;
202/// you can call `internalParse(T, ...)` for each of the container's items.
203/// you can call `innerParse(T, ...)` for each of the container's items.
203204/// Note that `null` fields are not allowed on the `options` when calling this function.
204205/// (The `options` you get in your `jsonParse` method has no `null` fields.)
205206pub fn innerParse(
......@@ -528,7 +529,12 @@ fn internalParseArray(
528529 return r;
529530}
530531
531fn internalParseFromValue(
532/// This is an internal function called recursively
533/// during the implementation of `parseFromValueLeaky`.
534/// It is exposed primarily to enable custom `jsonParseFromValue()` methods to call back into the `parseFromValue*` system,
535/// such as if you're implementing a custom container of type `T`;
536/// you can call `innerParseFromValue(T, ...)` for each of the container's items.
537pub fn innerParseFromValue(
532538 comptime T: type,
533539 allocator: Allocator,
534540 source: Value,
......@@ -571,7 +577,7 @@ fn internalParseFromValue(
571577 .Optional => |optionalInfo| {
572578 switch (source) {
573579 .null => return null,
574 else => return try internalParseFromValue(optionalInfo.child, allocator, source, options),
580 else => return try innerParseFromValue(optionalInfo.child, allocator, source, options),
575581 }
576582 },
577583 .Enum => {
......@@ -609,7 +615,7 @@ fn internalParseFromValue(
609615 return @unionInit(T, u_field.name, {});
610616 }
611617 // Recurse.
612 return @unionInit(T, u_field.name, try internalParseFromValue(u_field.type, allocator, kv.value_ptr.*, options));
618 return @unionInit(T, u_field.name, try innerParseFromValue(u_field.type, allocator, kv.value_ptr.*, options));
613619 }
614620 }
615621 // Didn't match anything.
......@@ -623,7 +629,7 @@ fn internalParseFromValue(
623629
624630 var r: T = undefined;
625631 inline for (0..structInfo.fields.len, source.array.items) |i, item| {
626 r[i] = try internalParseFromValue(structInfo.fields[i].type, allocator, item, options);
632 r[i] = try innerParseFromValue(structInfo.fields[i].type, allocator, item, options);
627633 }
628634
629635 return r;
......@@ -645,19 +651,8 @@ fn internalParseFromValue(
645651 inline for (structInfo.fields, 0..) |field, i| {
646652 if (field.is_comptime) @compileError("comptime fields are not supported: " ++ @typeName(T) ++ "." ++ field.name);
647653 if (std.mem.eql(u8, field.name, field_name)) {
648 if (fields_seen[i]) {
649 switch (options.duplicate_field_behavior) {
650 .use_first => {
651 // Parse and ignore the redundant value.
652 // We don't want to skip the value, because we want type checking.
653 _ = try internalParseFromValue(field.type, allocator, kv.value_ptr.*, options);
654 break;
655 },
656 .@"error" => return error.DuplicateField,
657 .use_last => {},
658 }
659 }
660 @field(r, field.name) = try internalParseFromValue(field.type, allocator, kv.value_ptr.*, options);
654 assert(!fields_seen[i]); // Can't have duplicate keys in a Value.object.
655 @field(r, field.name) = try innerParseFromValue(field.type, allocator, kv.value_ptr.*, options);
661656 fields_seen[i] = true;
662657 break;
663658 }
......@@ -674,7 +669,7 @@ fn internalParseFromValue(
674669 switch (source) {
675670 .array => |array| {
676671 // Typical array.
677 return internalParseArrayFromArrayValue(T, arrayInfo.child, arrayInfo.len, allocator, array, options);
672 return innerParseArrayFromArrayValue(T, arrayInfo.child, arrayInfo.len, allocator, array, options);
678673 },
679674 .string => |s| {
680675 if (arrayInfo.child != u8) return error.UnexpectedToken;
......@@ -694,7 +689,7 @@ fn internalParseFromValue(
694689 .Vector => |vecInfo| {
695690 switch (source) {
696691 .array => |array| {
697 return internalParseArrayFromArrayValue(T, vecInfo.child, vecInfo.len, allocator, array, options);
692 return innerParseArrayFromArrayValue(T, vecInfo.child, vecInfo.len, allocator, array, options);
698693 },
699694 else => return error.UnexpectedToken,
700695 }
......@@ -704,7 +699,7 @@ fn internalParseFromValue(
704699 switch (ptrInfo.size) {
705700 .One => {
706701 const r: *ptrInfo.child = try allocator.create(ptrInfo.child);
707 r.* = try internalParseFromValue(ptrInfo.child, allocator, source, options);
702 r.* = try innerParseFromValue(ptrInfo.child, allocator, source, options);
708703 return r;
709704 },
710705 .Slice => {
......@@ -716,7 +711,7 @@ fn internalParseFromValue(
716711 try allocator.alloc(ptrInfo.child, array.items.len);
717712
718713 for (array.items, r) |item, *dest| {
719 dest.* = try internalParseFromValue(ptrInfo.child, allocator, item, options);
714 dest.* = try innerParseFromValue(ptrInfo.child, allocator, item, options);
720715 }
721716
722717 return r;
......@@ -743,7 +738,7 @@ fn internalParseFromValue(
743738 }
744739}
745740
746fn internalParseArrayFromArrayValue(
741fn innerParseArrayFromArrayValue(
747742 comptime T: type,
748743 comptime Child: type,
749744 comptime len: comptime_int,
......@@ -755,7 +750,7 @@ fn internalParseArrayFromArrayValue(
755750
756751 var r: T = undefined;
757752 for (array.items, 0..) |item, i| {
758 r[i] = try internalParseFromValue(Child, allocator, item, options);
753 r[i] = try innerParseFromValue(Child, allocator, item, options);
759754 }
760755
761756 return r;