authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-10-10 23:22:18-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2019-10-10 23:25:40-04:00
log01b2c291d55e9f1b02c877537ba2b1b48079bb8b
tree3301e2f2770c9682304ecdd97686a55e3427e117
parent7b20205e689de78aea624639c356a10aff26b75b
signaturelock-open Commit is signed but in an unrecognized format.

miscellaneous improvements to generated docs

* introduce std.json.WriteStream API for writing json data to a stream * add WIP tools/merge_anal_dumps.zig for merging multiple semantic analysis dumps into one. See #3028 * add std.json.Array, improves generated docs * add test for `std.process.argsAlloc`, improves test coverage and generated docs

5 files changed, 380 insertions(+), 7 deletions(-)

lib/std/io.zig+1-1
......@@ -508,7 +508,7 @@ pub fn BitInStream(endian: builtin.Endian, comptime Error: type) type {
508508 };
509509}
510510
511/// This is a simple OutStream that writes to a slice, and returns an error
511/// This is a simple OutStream that writes to a fixed buffer, and returns an error
512512/// when it runs out of space.
513513pub const SliceOutStream = struct {
514514 pub const Error = error{OutOfSpace};
lib/std/json.zig+56-6
......@@ -8,6 +8,8 @@ const testing = std.testing;
88const mem = std.mem;
99const maxInt = std.math.maxInt;
1010
11pub const WriteStream = @import("json/write_stream.zig").WriteStream;
12
1113// A single token slice into the parent string.
1214//
1315// Use `token.slice()` on the input at the current position to get the current slice.
......@@ -1001,6 +1003,7 @@ pub const ValueTree = struct {
10011003};
10021004
10031005pub const ObjectMap = StringHashMap(Value);
1006pub const Array = ArrayList(Value);
10041007
10051008pub const Value = union(enum) {
10061009 Null,
......@@ -1008,7 +1011,7 @@ pub const Value = union(enum) {
10081011 Integer: i64,
10091012 Float: f64,
10101013 String: []const u8,
1011 Array: ArrayList(Value),
1014 Array: Array,
10121015 Object: ObjectMap,
10131016
10141017 pub fn dump(self: Value) void {
......@@ -1134,7 +1137,7 @@ pub const Parser = struct {
11341137 state: State,
11351138 copy_strings: bool,
11361139 // Stores parent nodes and un-combined Values.
1137 stack: ArrayList(Value),
1140 stack: Array,
11381141
11391142 const State = enum {
11401143 ObjectKey,
......@@ -1148,7 +1151,7 @@ pub const Parser = struct {
11481151 .allocator = allocator,
11491152 .state = State.Simple,
11501153 .copy_strings = copy_strings,
1151 .stack = ArrayList(Value).init(allocator),
1154 .stack = Array.init(allocator),
11521155 };
11531156 }
11541157
......@@ -1210,7 +1213,7 @@ pub const Parser = struct {
12101213 p.state = State.ObjectKey;
12111214 },
12121215 Token.Id.ArrayBegin => {
1213 try p.stack.append(Value{ .Array = ArrayList(Value).init(allocator) });
1216 try p.stack.append(Value{ .Array = Array.init(allocator) });
12141217 p.state = State.ArrayValue;
12151218 },
12161219 Token.Id.String => {
......@@ -1260,7 +1263,7 @@ pub const Parser = struct {
12601263 p.state = State.ObjectKey;
12611264 },
12621265 Token.Id.ArrayBegin => {
1263 try p.stack.append(Value{ .Array = ArrayList(Value).init(allocator) });
1266 try p.stack.append(Value{ .Array = Array.init(allocator) });
12641267 p.state = State.ArrayValue;
12651268 },
12661269 Token.Id.String => {
......@@ -1289,7 +1292,7 @@ pub const Parser = struct {
12891292 p.state = State.ObjectKey;
12901293 },
12911294 Token.Id.ArrayBegin => {
1292 try p.stack.append(Value{ .Array = ArrayList(Value).init(allocator) });
1295 try p.stack.append(Value{ .Array = Array.init(allocator) });
12931296 p.state = State.ArrayValue;
12941297 },
12951298 Token.Id.String => {
......@@ -1405,3 +1408,50 @@ test "json.parser.dynamic" {
14051408test "import more json tests" {
14061409 _ = @import("json/test.zig");
14071410}
1411
1412test "write json then parse it" {
1413 var out_buffer: [1000]u8 = undefined;
1414
1415 var slice_out_stream = std.io.SliceOutStream.init(&out_buffer);
1416 const out_stream = &slice_out_stream.stream;
1417 var jw = WriteStream(@typeOf(out_stream).Child, 4).init(out_stream);
1418
1419 try jw.beginObject();
1420
1421 try jw.objectField("f");
1422 try jw.emitBool(false);
1423
1424 try jw.objectField("t");
1425 try jw.emitBool(true);
1426
1427 try jw.objectField("int");
1428 try jw.emitNumber(i32(1234));
1429
1430 try jw.objectField("array");
1431 try jw.beginArray();
1432
1433 try jw.arrayElem();
1434 try jw.emitNull();
1435
1436 try jw.arrayElem();
1437 try jw.emitNumber(f64(12.34));
1438
1439 try jw.endArray();
1440
1441 try jw.objectField("str");
1442 try jw.emitString("hello");
1443
1444 try jw.endObject();
1445
1446 var mem_buffer: [1024 * 20]u8 = undefined;
1447 const allocator = &std.heap.FixedBufferAllocator.init(&mem_buffer).allocator;
1448 var parser = Parser.init(allocator, false);
1449 const tree = try parser.parse(slice_out_stream.getWritten());
1450
1451 testing.expect(tree.root.Object.get("f").?.value.Bool == false);
1452 testing.expect(tree.root.Object.get("t").?.value.Bool == true);
1453 testing.expect(tree.root.Object.get("int").?.value.Integer == 1234);
1454 testing.expect(tree.root.Object.get("array").?.value.Array.at(0).Null == {});
1455 testing.expect(tree.root.Object.get("array").?.value.Array.at(1).Float == 12.34);
1456 testing.expect(mem.eql(u8, tree.root.Object.get("str").?.value.String, "hello"));
1457}
lib/std/json/write_stream.zig created+211
......@@ -0,0 +1,211 @@
1const std = @import("../std.zig");
2const assert = std.debug.assert;
3const maxInt = std.math.maxInt;
4
5const State = enum {
6 Complete,
7 Value,
8 ArrayStart,
9 Array,
10 ObjectStart,
11 Object,
12};
13
14/// Writes JSON ([RFC8259](https://tools.ietf.org/html/rfc8259)) formatted data
15/// to a stream. `max_depth` is a comptime-known upper bound on the nesting depth.
16/// TODO A future iteration of this API will allow passing `null` for this value,
17/// and disable safety checks in release builds.
18pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type {
19 return struct {
20 const Self = @This();
21
22 pub const Stream = OutStream;
23
24 /// The string used for indenting.
25 one_indent: []const u8 = " ",
26
27 /// The string used as a newline character.
28 newline: []const u8 = "\n",
29
30 stream: *OutStream,
31 state_index: usize,
32 state: [max_depth]State,
33
34 pub fn init(stream: *OutStream) Self {
35 var self = Self{
36 .stream = stream,
37 .state_index = 1,
38 .state = undefined,
39 };
40 self.state[0] = .Complete;
41 self.state[1] = .Value;
42 return self;
43 }
44
45 pub fn beginArray(self: *Self) !void {
46 assert(self.state[self.state_index] == State.Value); // need to call arrayElem or objectField
47 try self.stream.writeByte('[');
48 self.state[self.state_index] = State.ArrayStart;
49 }
50
51 pub fn beginObject(self: *Self) !void {
52 assert(self.state[self.state_index] == State.Value); // need to call arrayElem or objectField
53 try self.stream.writeByte('{');
54 self.state[self.state_index] = State.ObjectStart;
55 }
56
57 pub fn arrayElem(self: *Self) !void {
58 const state = self.state[self.state_index];
59 switch (state) {
60 .Complete => unreachable,
61 .Value => unreachable,
62 .ObjectStart => unreachable,
63 .Object => unreachable,
64 .Array, .ArrayStart => {
65 if (state == .Array) {
66 try self.stream.writeByte(',');
67 }
68 self.state[self.state_index] = .Array;
69 self.pushState(.Value);
70 try self.indent();
71 },
72 }
73 }
74
75 pub fn objectField(self: *Self, name: []const u8) !void {
76 const state = self.state[self.state_index];
77 switch (state) {
78 .Complete => unreachable,
79 .Value => unreachable,
80 .ArrayStart => unreachable,
81 .Array => unreachable,
82 .Object, .ObjectStart => {
83 if (state == .Object) {
84 try self.stream.writeByte(',');
85 }
86 self.state[self.state_index] = .Object;
87 self.pushState(.Value);
88 try self.indent();
89 try self.writeEscapedString(name);
90 try self.stream.write(": ");
91 },
92 }
93 }
94
95 pub fn endArray(self: *Self) !void {
96 switch (self.state[self.state_index]) {
97 .Complete => unreachable,
98 .Value => unreachable,
99 .ObjectStart => unreachable,
100 .Object => unreachable,
101 .ArrayStart => {
102 try self.stream.writeByte(']');
103 self.popState();
104 },
105 .Array => {
106 try self.indent();
107 self.popState();
108 try self.stream.writeByte(']');
109 },
110 }
111 }
112
113 pub fn endObject(self: *Self) !void {
114 switch (self.state[self.state_index]) {
115 .Complete => unreachable,
116 .Value => unreachable,
117 .ArrayStart => unreachable,
118 .Array => unreachable,
119 .ObjectStart => {
120 try self.stream.writeByte('}');
121 self.popState();
122 },
123 .Object => {
124 try self.indent();
125 self.popState();
126 try self.stream.writeByte('}');
127 },
128 }
129 }
130
131 pub fn emitNull(self: *Self) !void {
132 assert(self.state[self.state_index] == State.Value);
133 try self.stream.write("null");
134 self.popState();
135 }
136
137 pub fn emitBool(self: *Self, value: bool) !void {
138 assert(self.state[self.state_index] == State.Value);
139 if (value) {
140 try self.stream.write("true");
141 } else {
142 try self.stream.write("false");
143 }
144 self.popState();
145 }
146
147 pub fn emitNumber(
148 self: *Self,
149 /// An integer, float, or `std.math.BigInt`. Emitted as a bare number if it fits losslessly
150 /// in a IEEE 754 double float, otherwise emitted as a string to the full precision.
151 value: var,
152 ) !void {
153 assert(self.state[self.state_index] == State.Value);
154 switch (@typeInfo(@typeOf(value))) {
155 .Int => |info| if (info.bits < 53 or (value < 4503599627370496 and value > -4503599627370496)) {
156 try self.stream.print("{}", value);
157 self.popState();
158 return;
159 },
160 .Float => if (@floatCast(f64, value) == value) {
161 try self.stream.print("{}", value);
162 self.popState();
163 return;
164 },
165 else => {},
166 }
167 try self.stream.print("\"{}\"", value);
168 self.popState();
169 }
170
171 pub fn emitString(self: *Self, string: []const u8) !void {
172 try self.writeEscapedString(string);
173 self.popState();
174 }
175
176 fn writeEscapedString(self: *Self, string: []const u8) !void {
177 try self.stream.writeByte('"');
178 for (string) |s| {
179 switch (s) {
180 '"' => try self.stream.write("\\\""),
181 '\t' => try self.stream.write("\\t"),
182 '\r' => try self.stream.write("\\r"),
183 '\n' => try self.stream.write("\\n"),
184 8 => try self.stream.write("\\b"),
185 12 => try self.stream.write("\\f"),
186 '\\' => try self.stream.write("\\\\"),
187 else => try self.stream.writeByte(s),
188 }
189 }
190 try self.stream.writeByte('"');
191 }
192
193 fn indent(self: *Self) !void {
194 assert(self.state_index >= 1);
195 try self.stream.write(self.newline);
196 var i: usize = 0;
197 while (i < self.state_index - 1) : (i += 1) {
198 try self.stream.write(self.one_indent);
199 }
200 }
201
202 fn pushState(self: *Self, state: State) void {
203 self.state_index += 1;
204 self.state[self.state_index] = state;
205 }
206
207 fn popState(self: *Self) void {
208 self.state_index -= 1;
209 }
210 };
211}
lib/std/os/test.zig+5
......@@ -232,3 +232,8 @@ test "pipe" {
232232 os.close(fds[1]);
233233 os.close(fds[0]);
234234}
235
236test "argsAlloc" {
237 var args = try std.process.argsAlloc(std.heap.direct_allocator);
238 std.heap.direct_allocator.free(args);
239}
tools/merge_anal_dumps.zig created+107
......@@ -0,0 +1,107 @@
1const builtin = @import("builtin");
2const std = @import("std");
3const json = std.json;
4const mem = std.mem;
5
6pub fn main() anyerror!void {
7 var arena = std.heap.ArenaAllocator.init(std.heap.direct_allocator);
8 defer arena.deinit();
9
10 const allocator = &arena.allocator;
11
12 const args = try std.process.argsAlloc(allocator);
13
14 var parser: json.Parser = undefined;
15 var dump = Dump.init(allocator);
16 for (args[1..]) |arg| {
17 parser = json.Parser.init(allocator, false);
18 const json_text = try std.io.readFileAlloc(allocator, arg);
19 const tree = try parser.parse(json_text);
20 try dump.mergeJson(tree.root);
21 }
22
23 const stdout = try std.io.getStdOut();
24 try dump.render(&stdout.outStream().stream);
25}
26
27const Dump = struct {
28 zig_id: ?[]const u8 = null,
29 zig_version: ?[]const u8 = null,
30 root_name: ?[]const u8 = null,
31 targets: std.ArrayList([]const u8),
32 files_list: std.ArrayList([]const u8),
33 files_map: std.StringHashMap(usize),
34
35 fn init(allocator: *mem.Allocator) Dump {
36 return Dump{
37 .targets = std.ArrayList([]const u8).init(allocator),
38 .files_list = std.ArrayList([]const u8).init(allocator),
39 .files_map = std.StringHashMap(usize).init(allocator),
40 };
41 }
42
43 fn mergeJson(self: *Dump, root: json.Value) !void {
44 const params = &root.Object.get("params").?.value.Object;
45 const zig_id = params.get("zigId").?.value.String;
46 const zig_version = params.get("zigVersion").?.value.String;
47 const root_name = params.get("rootName").?.value.String;
48 try mergeSameStrings(&self.zig_id, zig_id);
49 try mergeSameStrings(&self.zig_version, zig_version);
50 try mergeSameStrings(&self.root_name, root_name);
51
52 const target = params.get("target").?.value.String;
53 try self.targets.append(target);
54
55 // Merge files
56 const other_files = root.Object.get("files").?.value.Array.toSliceConst();
57 var other_file_to_mine = std.AutoHashMap(usize, usize).init(self.a());
58 for (other_files) |other_file, i| {
59 const gop = try self.files_map.getOrPut(other_file.String);
60 if (gop.found_existing) {
61 try other_file_to_mine.putNoClobber(i, gop.kv.value);
62 } else {
63 gop.kv.value = self.files_list.len;
64 try self.files_list.append(other_file.String);
65 }
66 }
67
68 const other_ast_nodes = root.Object.get("astNodes").?.value.Array.toSliceConst();
69 var other_ast_node_to_mine = std.AutoHashMap(usize, usize).init(self.a());
70 }
71
72 fn render(self: *Dump, stream: var) !void {
73 var jw = json.WriteStream(@typeOf(stream).Child, 10).init(stream);
74 try jw.beginObject();
75
76 try jw.objectField("typeKinds");
77 try jw.beginArray();
78 inline for (@typeInfo(builtin.TypeId).Enum.fields) |field| {
79 try jw.arrayElem();
80 try jw.emitString(field.name);
81 }
82 try jw.endArray();
83
84 try jw.objectField("files");
85 try jw.beginArray();
86 for (self.files_list.toSliceConst()) |file| {
87 try jw.arrayElem();
88 try jw.emitString(file);
89 }
90 try jw.endArray();
91
92 try jw.endObject();
93 }
94
95 fn a(self: Dump) *mem.Allocator {
96 return self.targets.allocator;
97 }
98
99 fn mergeSameStrings(opt_dest: *?[]const u8, src: []const u8) !void {
100 if (opt_dest.*) |dest| {
101 if (!mem.eql(u8, dest, src))
102 return error.MismatchedDumps;
103 } else {
104 opt_dest.* = src;
105 }
106 }
107};