Commit 9ea04f4f1ca2507c75f1d276c37028aa53bfa201 | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-03-31 22:12:22+02:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2023-04-01 08:31:16+02:00
log	9ea04f4f1ca2507c75f1d276c37028aa53bfa201
tree	31b50a83ee65ee0304d78f024f985b71aaf3014b
parent	9cb2919d500520d280e09d0990809d538aa32b56

tapi: update yaml parser

https://github.com/kubkon/zig-yaml/commit/5de8b0b3a2cdb86f9a173118efa7e5e0747cca14

5 files changed, 1752 insertions(+), 1121 deletions(-)

src
- link
  - tapi
    Tokenizer.zig
    parse.zig
    parse
    test.zig
    yaml.zig
    yaml
    test.zig

src/link/tapi/Tokenizer.zig+338-223

...	@@ -1,7 +1,7 @@	...	@@ -1,7 +1,7 @@
1	const Tokenizer = @This();	1	const Tokenizer = @This();
2		2
3	const std = @import("std");	3	const std = @import("std");
4	const log = std.log.scoped(.tapi);	4	const log = std.log.scoped(.yaml);
5	const testing = std.testing;	5	const testing = std.testing;
6		6
7	buffer: []const u8,	7	buffer: []const u8,
...	@@ -13,29 +13,31 @@ pub const Token = struct {	...	@@ -13,29 +13,31 @@ pub const Token = struct {
13	end: usize,	13	end: usize,
14		14
15	pub const Id = enum {	15	pub const Id = enum {
16	Eof,	16	// zig fmt: off
17		17	eof,
18	NewLine,	18
19	DocStart, // ---	19	new_line,
20	DocEnd, // ...	20	doc_start, // ---
21	SeqItemInd, // -	21	doc_end, // ...
22	MapValueInd, // :	22	seq_item_ind, // -
23	FlowMapStart, // {	23	map_value_ind, // :
24	FlowMapEnd, // }	24	flow_map_start, // {
25	FlowSeqStart, // [	25	flow_map_end, // }
26	FlowSeqEnd, // ]	26	flow_seq_start, // [
27		27	flow_seq_end, // ]
28	Comma,	28
29	Space,	29	comma,
30	Tab,	30	space,
31	Comment, // #	31	tab,
32	Alias, // *	32	comment, // #
33	Anchor, // &	33	alias, // *
34	Tag, // !	34	anchor, // &
35	SingleQuote, // '	35	tag, // !
36	DoubleQuote, // "	36
37		37	single_quoted, // '...'
38	Literal,	38	double_quoted, // "..."
		39	literal,
		40	// zig fmt: on
39	};	41	};
40	};	42	};
41		43
...	@@ -45,8 +47,8 @@ pub const TokenIterator = struct {	...	@@ -45,8 +47,8 @@ pub const TokenIterator = struct {
45	buffer: []const Token,	47	buffer: []const Token,
46	pos: TokenIndex = 0,	48	pos: TokenIndex = 0,
47		49
48	pub fn next(self: *TokenIterator) Token {	50	pub fn next(self: *TokenIterator) ?Token {
49	const token = self.buffer[self.pos];	51	const token = self.peek() orelse return null;
50	self.pos += 1;	52	self.pos += 1;
51	return token;	53	return token;
52	}	54	}
...	@@ -74,180 +76,212 @@ pub const TokenIterator = struct {	...	@@ -74,180 +76,212 @@ pub const TokenIterator = struct {
74	}	76	}
75	};	77	};
76		78
		79	fn stringMatchesPattern(comptime pattern: []const u8, slice: []const u8) bool {
		80	comptime var count: usize = 0;
		81	inline while (count < pattern.len) : (count += 1) {
		82	if (count >= slice.len) return false;
		83	const c = slice[count];
		84	if (pattern[count] != c) return false;
		85	}
		86	return true;
		87	}
		88
		89	fn matchesPattern(self: Tokenizer, comptime pattern: []const u8) bool {
		90	return stringMatchesPattern(pattern, self.buffer[self.index..]);
		91	}
		92
77	pub fn next(self: *Tokenizer) Token {	93	pub fn next(self: *Tokenizer) Token {
78	var result = Token{	94	var result = Token{
79	.id = .Eof,	95	.id = .eof,
80	.start = self.index,	96	.start = self.index,
81	.end = undefined,	97	.end = undefined,
82	};	98	};
83		99
84	var state: union(enum) {	100	var state: enum {
85	Start,	101	start,
86	NewLine,	102	new_line,
87	Space,	103	space,
88	Tab,	104	tab,
89	Hyphen: usize,	105	comment,
90	Dot: usize,	106	single_quoted,
91	Literal,	107	double_quoted,
92	} = .Start;	108	literal,
		109	} = .start;
93		110
94	while (self.index < self.buffer.len) : (self.index += 1) {	111	while (self.index < self.buffer.len) : (self.index += 1) {
95	const c = self.buffer[self.index];	112	const c = self.buffer[self.index];
96	switch (state) {	113	switch (state) {
97	.Start => switch (c) {	114	.start => switch (c) {
98	' ' => {	115	' ' => {
99	state = .Space;	116	state = .space;
100	},	117	},
101	'\t' => {	118	'\t' => {
102	state = .Tab;	119	state = .tab;
103	},	120	},
104	'\n' => {	121	'\n' => {
105	result.id = .NewLine;	122	result.id = .new_line;
106	self.index += 1;	123	self.index += 1;
107	break;	124	break;
108	},	125	},
109	'\r' => {	126	'\r' => {
110	state = .NewLine;	127	state = .new_line;
111	},	128	},
112	'-' => {	129
113	state = .{ .Hyphen = 1 };	130	'-' => if (self.matchesPattern("---")) {
		131	result.id = .doc_start;
		132	self.index += "---".len;
		133	break;
		134	} else if (self.matchesPattern("- ")) {
		135	result.id = .seq_item_ind;
		136	self.index += "- ".len;
		137	break;
		138	} else {
		139	state = .literal;
114	},	140	},
115	'.' => {	141
116	state = .{ .Dot = 1 };	142	'.' => if (self.matchesPattern("...")) {
		143	result.id = .doc_end;
		144	self.index += "...".len;
		145	break;
		146	} else {
		147	state = .literal;
117	},	148	},
		149
118	',' => {	150	',' => {
119	result.id = .Comma;	151	result.id = .comma;
120	self.index += 1;	152	self.index += 1;
121	break;	153	break;
122	},	154	},
123	'#' => {	155	'#' => {
124	result.id = .Comment;	156	state = .comment;
125	self.index += 1;
126	break;
127	},	157	},
128	'*' => {	158	'*' => {
129	result.id = .Alias;	159	result.id = .alias;
130	self.index += 1;	160	self.index += 1;
131	break;	161	break;
132	},	162	},
133	'&' => {	163	'&' => {
134	result.id = .Anchor;	164	result.id = .anchor;
135	self.index += 1;	165	self.index += 1;
136	break;	166	break;
137	},	167	},
138	'!' => {	168	'!' => {
139	result.id = .Tag;	169	result.id = .tag;
140	self.index += 1;
141	break;
142	},
143	'\'' => {
144	result.id = .SingleQuote;
145	self.index += 1;
146	break;
147	},
148	'"' => {
149	result.id = .DoubleQuote;
150	self.index += 1;	170	self.index += 1;
151	break;	171	break;
152	},	172	},
153	'[' => {	173	'[' => {
154	result.id = .FlowSeqStart;	174	result.id = .flow_seq_start;
155	self.index += 1;	175	self.index += 1;
156	break;	176	break;
157	},	177	},
158	']' => {	178	']' => {
159	result.id = .FlowSeqEnd;	179	result.id = .flow_seq_end;
160	self.index += 1;	180	self.index += 1;
161	break;	181	break;
162	},	182	},
163	':' => {	183	':' => {
164	result.id = .MapValueInd;	184	result.id = .map_value_ind;
165	self.index += 1;	185	self.index += 1;
166	break;	186	break;
167	},	187	},
168	'{' => {	188	'{' => {
169	result.id = .FlowMapStart;	189	result.id = .flow_map_start;
170	self.index += 1;	190	self.index += 1;
171	break;	191	break;
172	},	192	},
173	'}' => {	193	'}' => {
174	result.id = .FlowMapEnd;	194	result.id = .flow_map_end;
175	self.index += 1;	195	self.index += 1;
176	break;	196	break;
177	},	197	},
		198	'\'' => {
		199	state = .single_quoted;
		200	},
		201	'"' => {
		202	state = .double_quoted;
		203	},
178	else => {	204	else => {
179	state = .Literal;	205	state = .literal;
		206	},
		207	},
		208
		209	.comment => switch (c) {
		210	'\r', '\n' => {
		211	result.id = .comment;
		212	break;
180	},	213	},
		214	else => {},
181	},	215	},
182	.Space => switch (c) {	216
		217	.space => switch (c) {
183	' ' => {},	218	' ' => {},
184	else => {	219	else => {
185	result.id = .Space;	220	result.id = .space;
186	break;	221	break;
187	},	222	},
188	},	223	},
189	.Tab => switch (c) {	224
		225	.tab => switch (c) {
190	'\t' => {},	226	'\t' => {},
191	else => {	227	else => {
192	result.id = .Tab;	228	result.id = .tab;
193	break;	229	break;
194	},	230	},
195	},	231	},
196	.NewLine => switch (c) {	232
		233	.new_line => switch (c) {
197	'\n' => {	234	'\n' => {
198	result.id = .NewLine;	235	result.id = .new_line;
199	self.index += 1;	236	self.index += 1;
200	break;	237	break;
201	},	238	},
202	else => {}, // TODO this should be an error condition	239	else => {}, // TODO this should be an error condition
203	},	240	},
204	.Hyphen => \|*count\| switch (c) {	241
205	' ' => {	242	.single_quoted => switch (c) {
206	result.id = .SeqItemInd;	243	'\'' => if (!self.matchesPattern("''")) {
		244	result.id = .single_quoted;
207	self.index += 1;	245	self.index += 1;
208	break;	246	break;
		247	} else {
		248	self.index += "''".len - 1;
209	},	249	},
210	'-' => {	250	else => {},
211	count.* += 1;
212
213	if (count.* == 3) {
214	result.id = .DocStart;
215	self.index += 1;
216	break;
217	}
218	},
219	else => {
220	state = .Literal;
221	},
222	},	251	},
223	.Dot => \|*count\| switch (c) {
224	'.' => {
225	count.* += 1;
226		252
227	if (count.* == 3) {	253	.double_quoted => switch (c) {
228	result.id = .DocEnd;	254	'"' => {
		255	if (stringMatchesPattern("\\", self.buffer[self.index - 1 ..])) {
		256	self.index += 1;
		257	} else {
		258	result.id = .double_quoted;
229	self.index += 1;	259	self.index += 1;
230	break;	260	break;
231	}	261	}
232	},	262	},
233	else => {	263	else => {},
234	state = .Literal;
235	},
236	},	264	},
237	.Literal => switch (c) {	265
		266	.literal => switch (c) {
238	'\r', '\n', ' ', '\'', '"', ',', ':', ']', '}' => {	267	'\r', '\n', ' ', '\'', '"', ',', ':', ']', '}' => {
239	result.id = .Literal;	268	result.id = .literal;
240	break;	269	break;
241	},	270	},
242	else => {	271	else => {
243	result.id = .Literal;	272	result.id = .literal;
244	},	273	},
245	},	274	},
246	}	275	}
247	}	276	}
248		277
249	if (state == .Literal and result.id == .Eof) {	278	if (self.index >= self.buffer.len) {
250	result.id = .Literal;	279	switch (state) {
		280	.literal => {
		281	result.id = .literal;
		282	},
		283	else => {},
		284	}
251	}	285	}
252		286
253	result.end = self.index;	287	result.end = self.index;
...	@@ -263,22 +297,24 @@ fn testExpected(source: []const u8, expected: []const Token.Id) !void {	...	@@ -263,22 +297,24 @@ fn testExpected(source: []const u8, expected: []const Token.Id) !void {
263	.buffer = source,	297	.buffer = source,
264	};	298	};
265		299
266	var token_len: usize = 0;	300	var given = std.ArrayList(Token.Id).init(testing.allocator);
267	for (expected) \|exp\| {	301	defer given.deinit();
268	token_len += 1;	302
		303	while (true) {
269	const token = tokenizer.next();	304	const token = tokenizer.next();
270	try testing.expectEqual(exp, token.id);	305	try given.append(token.id);
		306	if (token.id == .eof) break;
271	}	307	}
272		308
273	while (tokenizer.next().id != .Eof) {	309	try testing.expectEqualSlices(Token.Id, expected, given.items);
274	token_len += 1; // consume all tokens	310	}
275	}
276		311
277	try testing.expectEqual(expected.len, token_len);	312	test {
		313	std.testing.refAllDecls(@This());
278	}	314	}
279		315
280	test "empty doc" {	316	test "empty doc" {
281	try testExpected("", &[_]Token.Id{.Eof});	317	try testExpected("", &[_]Token.Id{.eof});
282	}	318	}
283		319
284	test "empty doc with explicit markers" {	320	test "empty doc with explicit markers" {
...	@@ -286,7 +322,22 @@ test "empty doc with explicit markers" {	...	@@ -286,7 +322,22 @@ test "empty doc with explicit markers" {
286	\\---	322	\\---
287	\\...	323	\\...
288	, &[_]Token.Id{	324	, &[_]Token.Id{
289	.DocStart, .NewLine, .DocEnd, .Eof,	325	.doc_start, .new_line, .doc_end, .eof,
		326	});
		327	}
		328
		329	test "empty doc with explicit markers and a directive" {
		330	try testExpected(
		331	\\--- !tbd-v1
		332	\\...
		333	, &[_]Token.Id{
		334	.doc_start,
		335	.space,
		336	.tag,
		337	.literal,
		338	.new_line,
		339	.doc_end,
		340	.eof,
290	});	341	});
291	}	342	}
292		343
...	@@ -296,15 +347,15 @@ test "sequence of values" {	...	@@ -296,15 +347,15 @@ test "sequence of values" {
296	\\- 1	347	\\- 1
297	\\- 2	348	\\- 2
298	, &[_]Token.Id{	349	, &[_]Token.Id{
299	.SeqItemInd,	350	.seq_item_ind,
300	.Literal,	351	.literal,
301	.NewLine,	352	.new_line,
302	.SeqItemInd,	353	.seq_item_ind,
303	.Literal,	354	.literal,
304	.NewLine,	355	.new_line,
305	.SeqItemInd,	356	.seq_item_ind,
306	.Literal,	357	.literal,
307	.Eof,	358	.eof,
308	});	359	});
309	}	360	}
310		361
...	@@ -313,24 +364,24 @@ test "sequence of sequences" {	...	@@ -313,24 +364,24 @@ test "sequence of sequences" {
313	\\- [ val1, val2]	364	\\- [ val1, val2]
314	\\- [val3, val4 ]	365	\\- [val3, val4 ]
315	, &[_]Token.Id{	366	, &[_]Token.Id{
316	.SeqItemInd,	367	.seq_item_ind,
317	.FlowSeqStart,	368	.flow_seq_start,
318	.Space,	369	.space,
319	.Literal,	370	.literal,
320	.Comma,	371	.comma,
321	.Space,	372	.space,
322	.Literal,	373	.literal,
323	.FlowSeqEnd,	374	.flow_seq_end,
324	.NewLine,	375	.new_line,
325	.SeqItemInd,	376	.seq_item_ind,
326	.FlowSeqStart,	377	.flow_seq_start,
327	.Literal,	378	.literal,
328	.Comma,	379	.comma,
329	.Space,	380	.space,
330	.Literal,	381	.literal,
331	.Space,	382	.space,
332	.FlowSeqEnd,	383	.flow_seq_end,
333	.Eof,	384	.eof,
334	});	385	});
335	}	386	}
336		387
...	@@ -339,16 +390,16 @@ test "mappings" {	...	@@ -339,16 +390,16 @@ test "mappings" {
339	\\key1: value1	390	\\key1: value1
340	\\key2: value2	391	\\key2: value2
341	, &[_]Token.Id{	392	, &[_]Token.Id{
342	.Literal,	393	.literal,
343	.MapValueInd,	394	.map_value_ind,
344	.Space,	395	.space,
345	.Literal,	396	.literal,
346	.NewLine,	397	.new_line,
347	.Literal,	398	.literal,
348	.MapValueInd,	399	.map_value_ind,
349	.Space,	400	.space,
350	.Literal,	401	.literal,
351	.Eof,	402	.eof,
352	});	403	});
353	}	404	}
354		405
...	@@ -357,21 +408,21 @@ test "inline mapped sequence of values" {	...	@@ -357,21 +408,21 @@ test "inline mapped sequence of values" {
357	\\key : [ val1,	408	\\key : [ val1,
358	\\ val2 ]	409	\\ val2 ]
359	, &[_]Token.Id{	410	, &[_]Token.Id{
360	.Literal,	411	.literal,
361	.Space,	412	.space,
362	.MapValueInd,	413	.map_value_ind,
363	.Space,	414	.space,
364	.FlowSeqStart,	415	.flow_seq_start,
365	.Space,	416	.space,
366	.Literal,	417	.literal,
367	.Comma,	418	.comma,
368	.Space,	419	.space,
369	.NewLine,	420	.new_line,
370	.Space,	421	.space,
371	.Literal,	422	.literal,
372	.Space,	423	.space,
373	.FlowSeqEnd,	424	.flow_seq_end,
374	.Eof,	425	.eof,
375	});	426	});
376	}	427	}
377		428
...	@@ -388,52 +439,50 @@ test "part of tbd" {	...	@@ -388,52 +439,50 @@ test "part of tbd" {
388	\\install-name: '/usr/lib/libSystem.B.dylib'	439	\\install-name: '/usr/lib/libSystem.B.dylib'
389	\\...	440	\\...
390	, &[_]Token.Id{	441	, &[_]Token.Id{
391	.DocStart,	442	.doc_start,
392	.Space,	443	.space,
393	.Tag,	444	.tag,
394	.Literal,	445	.literal,
395	.NewLine,	446	.new_line,
396	.Literal,	447	.literal,
397	.MapValueInd,	448	.map_value_ind,
398	.Space,	449	.space,
399	.Literal,	450	.literal,
400	.NewLine,	451	.new_line,
401	.Literal,	452	.literal,
402	.MapValueInd,	453	.map_value_ind,
403	.Space,	454	.space,
404	.FlowSeqStart,	455	.flow_seq_start,
405	.Space,	456	.space,
406	.Literal,	457	.literal,
407	.Space,	458	.space,
408	.FlowSeqEnd,	459	.flow_seq_end,
409	.NewLine,	460	.new_line,
410	.NewLine,	461	.new_line,
411	.Literal,	462	.literal,
412	.MapValueInd,	463	.map_value_ind,
413	.NewLine,	464	.new_line,
414	.Space,	465	.space,
415	.SeqItemInd,	466	.seq_item_ind,
416	.Literal,	467	.literal,
417	.MapValueInd,	468	.map_value_ind,
418	.Space,	469	.space,
419	.Literal,	470	.literal,
420	.NewLine,	471	.new_line,
421	.Space,	472	.space,
422	.Literal,	473	.literal,
423	.MapValueInd,	474	.map_value_ind,
424	.Space,	475	.space,
425	.Literal,	476	.literal,
426	.NewLine,	477	.new_line,
427	.NewLine,	478	.new_line,
428	.Literal,	479	.literal,
429	.MapValueInd,	480	.map_value_ind,
430	.Space,	481	.space,
431	.SingleQuote,	482	.single_quoted,
432	.Literal,	483	.new_line,
433	.SingleQuote,	484	.doc_end,
434	.NewLine,	485	.eof,
435	.DocEnd,
436	.Eof,
437	});	486	});
438	}	487	}
439		488
...	@@ -443,18 +492,84 @@ test "Unindented list" {	...	@@ -443,18 +492,84 @@ test "Unindented list" {
443	\\- foo: 1	492	\\- foo: 1
444	\\c: 1	493	\\c: 1
445	, &[_]Token.Id{	494	, &[_]Token.Id{
446	.Literal,	495	.literal,
447	.MapValueInd,	496	.map_value_ind,
448	.NewLine,	497	.new_line,
449	.SeqItemInd,	498	.seq_item_ind,
450	.Literal,	499	.literal,
451	.MapValueInd,	500	.map_value_ind,
452	.Space,	501	.space,
453	.Literal,	502	.literal,
454	.NewLine,	503	.new_line,
455	.Literal,	504	.literal,
456	.MapValueInd,	505	.map_value_ind,
457	.Space,	506	.space,
458	.Literal,	507	.literal,
		508	.eof,
		509	});
		510	}
		511
		512	test "escape sequences" {
		513	try testExpected(
		514	\\a: 'here''s an apostrophe'
		515	\\b: "a newline\nand a\ttab"
		516	\\c: "\"here\" and there"
		517	, &[_]Token.Id{
		518	.literal,
		519	.map_value_ind,
		520	.space,
		521	.single_quoted,
		522	.new_line,
		523	.literal,
		524	.map_value_ind,
		525	.space,
		526	.double_quoted,
		527	.new_line,
		528	.literal,
		529	.map_value_ind,
		530	.space,
		531	.double_quoted,
		532	.eof,
		533	});
		534	}
		535
		536	test "comments" {
		537	try testExpected(
		538	\\key: # some comment about the key
		539	\\# first value
		540	\\- val1
		541	\\# second value
		542	\\- val2
		543	, &[_]Token.Id{
		544	.literal,
		545	.map_value_ind,
		546	.space,
		547	.comment,
		548	.new_line,
		549	.comment,
		550	.new_line,
		551	.seq_item_ind,
		552	.literal,
		553	.new_line,
		554	.comment,
		555	.new_line,
		556	.seq_item_ind,
		557	.literal,
		558	.eof,
		559	});
		560	}
		561
		562	test "quoted literals" {
		563	try testExpected(
		564	\\'#000000'
		565	\\'[000000'
		566	\\"&someString"
		567	, &[_]Token.Id{
		568	.single_quoted,
		569	.new_line,
		570	.single_quoted,
		571	.new_line,
		572	.double_quoted,
		573	.eof,
459	});	574	});
460	}	575	}

src/link/tapi/parse.zig+378-323

...	@@ -1,8 +1,7 @@	...	@@ -1,8 +1,7 @@
1	const std = @import("std");	1	const std = @import("std");
2	const assert = std.debug.assert;	2	const assert = std.debug.assert;
3	const log = std.log.scoped(.tapi);	3	const log = std.log.scoped(.yaml);
4	const mem = std.mem;	4	const mem = std.mem;
5	const testing = std.testing;
6		5
7	const Allocator = mem.Allocator;	6	const Allocator = mem.Allocator;
8	const Tokenizer = @import("Tokenizer.zig");	7	const Tokenizer = @import("Tokenizer.zig");
...	@@ -11,9 +10,9 @@ const TokenIndex = Tokenizer.TokenIndex;	...	@@ -11,9 +10,9 @@ const TokenIndex = Tokenizer.TokenIndex;
11	const TokenIterator = Tokenizer.TokenIterator;	10	const TokenIterator = Tokenizer.TokenIterator;
12		11
13	pub const ParseError = error{	12	pub const ParseError = error{
		13	InvalidEscapeSequence,
14	MalformedYaml,	14	MalformedYaml,
15	NestedDocuments,	15	NestedDocuments,
16	UnexpectedTag,
17	UnexpectedEof,	16	UnexpectedEof,
18	UnexpectedToken,	17	UnexpectedToken,
19	Unhandled,	18	Unhandled,
...	@@ -22,6 +21,8 @@ pub const ParseError = error{	...	@@ -22,6 +21,8 @@ pub const ParseError = error{
22	pub const Node = struct {	21	pub const Node = struct {
23	tag: Tag,	22	tag: Tag,
24	tree: *const Tree,	23	tree: *const Tree,
		24	start: TokenIndex,
		25	end: TokenIndex,
25		26
26	pub const Tag = enum {	27	pub const Tag = enum {
27	doc,	28	doc,
...	@@ -61,9 +62,12 @@ pub const Node = struct {	...	@@ -61,9 +62,12 @@ pub const Node = struct {
61	}	62	}
62		63
63	pub const Doc = struct {	64	pub const Doc = struct {
64	base: Node = Node{ .tag = Tag.doc, .tree = undefined },	65	base: Node = Node{
65	start: ?TokenIndex = null,	66	.tag = Tag.doc,
66	end: ?TokenIndex = null,	67	.tree = undefined,
		68	.start = undefined,
		69	.end = undefined,
		70	},
67	directive: ?TokenIndex = null,	71	directive: ?TokenIndex = null,
68	value: ?*Node = null,	72	value: ?*Node = null,
69		73
...	@@ -86,10 +90,8 @@ pub const Node = struct {	...	@@ -86,10 +90,8 @@ pub const Node = struct {
86	_ = fmt;	90	_ = fmt;
87	if (self.directive) \|id\| {	91	if (self.directive) \|id\| {
88	try std.fmt.format(writer, "{{ ", .{});	92	try std.fmt.format(writer, "{{ ", .{});
89	const directive = self.base.tree.tokens[id];	93	const directive = self.base.tree.getRaw(id, id);
90	try std.fmt.format(writer, ".directive = {s}, ", .{	94	try std.fmt.format(writer, ".directive = {s}, ", .{directive});
91	self.base.tree.source[directive.start..directive.end],
92	});
93	}	95	}
94	if (self.value) \|node\| {	96	if (self.value) \|node\| {
95	try std.fmt.format(writer, "{}", .{node});	97	try std.fmt.format(writer, "{}", .{node});
...	@@ -101,22 +103,27 @@ pub const Node = struct {	...	@@ -101,22 +103,27 @@ pub const Node = struct {
101	};	103	};
102		104
103	pub const Map = struct {	105	pub const Map = struct {
104	base: Node = Node{ .tag = Tag.map, .tree = undefined },	106	base: Node = Node{
105	start: ?TokenIndex = null,	107	.tag = Tag.map,
106	end: ?TokenIndex = null,	108	.tree = undefined,
		109	.start = undefined,
		110	.end = undefined,
		111	},
107	values: std.ArrayListUnmanaged(Entry) = .{},	112	values: std.ArrayListUnmanaged(Entry) = .{},
108		113
109	pub const base_tag: Node.Tag = .map;	114	pub const base_tag: Node.Tag = .map;
110		115
111	pub const Entry = struct {	116	pub const Entry = struct {
112	key: TokenIndex,	117	key: TokenIndex,
113	value: *Node,	118	value: ?*Node,
114	};	119	};
115		120
116	pub fn deinit(self: *Map, allocator: Allocator) void {	121	pub fn deinit(self: *Map, allocator: Allocator) void {
117	for (self.values.items) \|entry\| {	122	for (self.values.items) \|entry\| {
118	entry.value.deinit(allocator);	123	if (entry.value) \|value\| {
119	allocator.destroy(entry.value);	124	value.deinit(allocator);
		125	allocator.destroy(value);
		126	}
120	}	127	}
121	self.values.deinit(allocator);	128	self.values.deinit(allocator);
122	}	129	}
...	@@ -131,20 +138,24 @@ pub const Node = struct {	...	@@ -131,20 +138,24 @@ pub const Node = struct {
131	_ = fmt;	138	_ = fmt;
132	try std.fmt.format(writer, "{{ ", .{});	139	try std.fmt.format(writer, "{{ ", .{});
133	for (self.values.items) \|entry\| {	140	for (self.values.items) \|entry\| {
134	const key = self.base.tree.tokens[entry.key];	141	const key = self.base.tree.getRaw(entry.key, entry.key);
135	try std.fmt.format(writer, "{s} => {}, ", .{	142	if (entry.value) \|value\| {
136	self.base.tree.source[key.start..key.end],	143	try std.fmt.format(writer, "{s} => {}, ", .{ key, value });
137	entry.value,	144	} else {
138	});	145	try std.fmt.format(writer, "{s} => null, ", .{key});
		146	}
139	}	147	}
140	return std.fmt.format(writer, " }}", .{});	148	return std.fmt.format(writer, " }}", .{});
141	}	149	}
142	};	150	};
143		151
144	pub const List = struct {	152	pub const List = struct {
145	base: Node = Node{ .tag = Tag.list, .tree = undefined },	153	base: Node = Node{
146	start: ?TokenIndex = null,	154	.tag = Tag.list,
147	end: ?TokenIndex = null,	155	.tree = undefined,
		156	.start = undefined,
		157	.end = undefined,
		158	},
148	values: std.ArrayListUnmanaged(*Node) = .{},	159	values: std.ArrayListUnmanaged(*Node) = .{},
149		160
150	pub const base_tag: Node.Tag = .list;	161	pub const base_tag: Node.Tag = .list;
...	@@ -174,15 +185,18 @@ pub const Node = struct {	...	@@ -174,15 +185,18 @@ pub const Node = struct {
174	};	185	};
175		186
176	pub const Value = struct {	187	pub const Value = struct {
177	base: Node = Node{ .tag = Tag.value, .tree = undefined },	188	base: Node = Node{
178	start: ?TokenIndex = null,	189	.tag = Tag.value,
179	end: ?TokenIndex = null,	190	.tree = undefined,
		191	.start = undefined,
		192	.end = undefined,
		193	},
		194	string_value: std.ArrayListUnmanaged(u8) = .{},
180		195
181	pub const base_tag: Node.Tag = .value;	196	pub const base_tag: Node.Tag = .value;
182		197
183	pub fn deinit(self: *Value, allocator: Allocator) void {	198	pub fn deinit(self: *Value, allocator: Allocator) void {
184	_ = self;	199	self.string_value.deinit(allocator);
185	_ = allocator;
186	}	200	}
187		201
188	pub fn format(	202	pub fn format(
...	@@ -193,11 +207,8 @@ pub const Node = struct {	...	@@ -193,11 +207,8 @@ pub const Node = struct {
193	) !void {	207	) !void {
194	_ = options;	208	_ = options;
195	_ = fmt;	209	_ = fmt;
196	const start = self.base.tree.tokens[self.start.?];	210	const raw = self.base.tree.getRaw(self.base.start, self.base.end);
197	const end = self.base.tree.tokens[self.end.?];	211	return std.fmt.format(writer, "{s}", .{raw});
198	return std.fmt.format(writer, "{s}", .{
199	self.base.tree.source[start.start..end.end],
200	});
201	}	212	}
202	};	213	};
203	};	214	};
...	@@ -233,6 +244,21 @@ pub const Tree = struct {	...	@@ -233,6 +244,21 @@ pub const Tree = struct {
233	self.docs.deinit(self.allocator);	244	self.docs.deinit(self.allocator);
234	}	245	}
235		246
		247	pub fn getDirective(self: Tree, doc_index: usize) ?[]const u8 {
		248	assert(doc_index < self.docs.items.len);
		249	const doc = self.docs.items[doc_index].cast(Node.Doc) orelse return null;
		250	const id = doc.directive orelse return null;
		251	return self.getRaw(id, id);
		252	}
		253
		254	pub fn getRaw(self: Tree, start: TokenIndex, end: TokenIndex) []const u8 {
		255	assert(start <= end);
		256	assert(start < self.tokens.len and end < self.tokens.len);
		257	const start_token = self.tokens[start];
		258	const end_token = self.tokens[end];
		259	return self.source[start_token.start..end_token.end];
		260	}
		261
236	pub fn parse(self: *Tree, source: []const u8) !void {	262	pub fn parse(self: *Tree, source: []const u8) !void {
237	var tokenizer = Tokenizer{ .buffer = source };	263	var tokenizer = Tokenizer{ .buffer = source };
238	var tokens = std.ArrayList(Token).init(self.allocator);	264	var tokens = std.ArrayList(Token).init(self.allocator);
...	@@ -252,8 +278,8 @@ pub const Tree = struct {	...	@@ -252,8 +278,8 @@ pub const Tree = struct {
252	});	278	});
253		279
254	switch (token.id) {	280	switch (token.id) {
255	.Eof => break,	281	.eof => break,
256	.NewLine => {	282	.new_line => {
257	line += 1;	283	line += 1;
258	prev_line_last_col = token.end;	284	prev_line_last_col = token.end;
259	},	285	},
...	@@ -272,20 +298,20 @@ pub const Tree = struct {	...	@@ -272,20 +298,20 @@ pub const Tree = struct {
272	.line_cols = &self.line_cols,	298	.line_cols = &self.line_cols,
273	};	299	};
274		300
275	while (true) {	301	parser.eatCommentsAndSpace(&.{});
276	if (parser.token_it.peek() == null) return;
277		302
278	const pos = parser.token_it.pos;	303	while (true) {
279	const token = parser.token_it.next();	304	parser.eatCommentsAndSpace(&.{});
		305	const token = parser.token_it.next() orelse break;
280		306
281	log.debug("Next token: {}, {}", .{ pos, token });	307	log.debug("(main) next {s}@{d}", .{ @tagName(token.id), parser.token_it.pos - 1 });
282		308
283	switch (token.id) {	309	switch (token.id) {
284	.Space, .Comment, .NewLine => {},	310	.eof => break,
285	.Eof => break,
286	else => {	311	else => {
287	const doc = try parser.doc(pos);	312	parser.token_it.seekBy(-1);
288	try self.docs.append(self.allocator, &doc.base);	313	const doc = try parser.doc();
		314	try self.docs.append(self.allocator, doc);
289	},	315	},
290	}	316	}
291	}	317	}
...	@@ -298,355 +324,308 @@ const Parser = struct {	...	@@ -298,355 +324,308 @@ const Parser = struct {
298	token_it: *TokenIterator,	324	token_it: *TokenIterator,
299	line_cols: *const std.AutoHashMap(TokenIndex, LineCol),	325	line_cols: *const std.AutoHashMap(TokenIndex, LineCol),
300		326
301	fn doc(self: Parser, start: TokenIndex) ParseError!Node.Doc {	327	fn value(self: Parser) ParseError!?Node {
		328	self.eatCommentsAndSpace(&.{});
		329
		330	const pos = self.token_it.pos;
		331	const token = self.token_it.next() orelse return error.UnexpectedEof;
		332
		333	log.debug(" next {s}@{d}", .{ @tagName(token.id), pos });
		334
		335	switch (token.id) {
		336	.literal => if (self.eatToken(.map_value_ind, &.{ .new_line, .comment })) \|_\| {
		337	// map
		338	self.token_it.seekTo(pos);
		339	return self.map();
		340	} else {
		341	// leaf value
		342	self.token_it.seekTo(pos);
		343	return self.leaf_value();
		344	},
		345	.single_quoted, .double_quoted => {
		346	// leaf value
		347	self.token_it.seekBy(-1);
		348	return self.leaf_value();
		349	},
		350	.seq_item_ind => {
		351	// list
		352	self.token_it.seekBy(-1);
		353	return self.list();
		354	},
		355	.flow_seq_start => {
		356	// list
		357	self.token_it.seekBy(-1);
		358	return self.list_bracketed();
		359	},
		360	else => return null,
		361	}
		362	}
		363
		364	fn doc(self: Parser) ParseError!Node {
302	const node = try self.allocator.create(Node.Doc);	365	const node = try self.allocator.create(Node.Doc);
303	errdefer self.allocator.destroy(node);	366	errdefer self.allocator.destroy(node);
304	node.* = .{ .start = start };	367	node.* = .{};
305	node.base.tree = self.tree;	368	node.base.tree = self.tree;
		369	node.base.start = self.token_it.pos;
306		370
307	self.token_it.seekTo(start);	371	log.debug("(doc) begin {s}@{d}", .{ @tagName(self.tree.tokens[node.base.start].id), node.base.start });
308
309	log.debug("Doc start: {}, {}", .{ start, self.tree.tokens[start] });
310		372
311	const explicit_doc: bool = if (self.eatToken(.DocStart)) \|_\| explicit_doc: {	373	// Parse header
312	if (self.eatToken(.Tag)) \|_\| {	374	const explicit_doc: bool = if (self.eatToken(.doc_start, &.{})) \|doc_pos\| explicit_doc: {
313	node.directive = try self.expectToken(.Literal);	375	if (self.getCol(doc_pos) > 0) return error.MalformedYaml;
		376	if (self.eatToken(.tag, &.{ .new_line, .comment })) \|_\| {
		377	node.directive = try self.expectToken(.literal, &.{ .new_line, .comment });
314	}	378	}
315	_ = try self.expectToken(.NewLine);
316	break :explicit_doc true;	379	break :explicit_doc true;
317	} else false;	380	} else false;
318		381
319	while (true) {	382	// Parse value
320	const pos = self.token_it.pos;	383	node.value = try self.value();
321	const token = self.token_it.next();	384	if (node.value == null) {
322		385	self.token_it.seekBy(-1);
323	log.debug("Next token: {}, {}", .{ pos, token });	386	}
		387	errdefer if (node.value) \|val\| {
		388	val.deinit(self.allocator);
		389	self.allocator.destroy(val);
		390	};
324		391
325	switch (token.id) {	392	// Parse footer
326	.Tag => {	393	footer: {
327	return error.UnexpectedTag;	394	if (self.eatToken(.doc_end, &.{})) \|pos\| {
328	},	395	if (!explicit_doc) return error.UnexpectedToken;
329	.Literal, .SingleQuote, .DoubleQuote => {	396	if (self.getCol(pos) > 0) return error.MalformedYaml;
330	_ = try self.expectToken(.MapValueInd);	397	node.base.end = pos;
331	const map_node = try self.map(pos);	398	break :footer;
332	node.value = &map_node.base;	399	}
333	},	400	if (self.eatToken(.doc_start, &.{})) \|pos\| {
334	.SeqItemInd => {	401	if (!explicit_doc) return error.UnexpectedToken;
335	const list_node = try self.list(pos);	402	if (self.getCol(pos) > 0) return error.MalformedYaml;
336	node.value = &list_node.base;	403	self.token_it.seekBy(-1);
337	},	404	node.base.end = pos - 1;
338	.FlowSeqStart => {	405	break :footer;
339	const list_node = try self.list_bracketed(pos);	406	}
340	node.value = &list_node.base;	407	if (self.eatToken(.eof, &.{})) \|pos\| {
341	},	408	node.base.end = pos - 1;
342	.DocEnd => {	409	break :footer;
343	if (explicit_doc) break;
344	return error.UnexpectedToken;
345	},
346	.DocStart, .Eof => {
347	self.token_it.seekBy(-1);
348	break;
349	},
350	else => {
351	return error.UnexpectedToken;
352	},
353	}	410	}
		411	return error.UnexpectedToken;
354	}	412	}
355		413
356	node.end = self.token_it.pos - 1;	414	log.debug("(doc) end {s}@{d}", .{ @tagName(self.tree.tokens[node.base.end].id), node.base.end });
357		415
358	log.debug("Doc end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] });	416	return &node.base;
359
360	return node;
361	}	417	}
362		418
363	fn map(self: Parser, start: TokenIndex) ParseError!Node.Map {	419	fn map(self: Parser) ParseError!Node {
364	const node = try self.allocator.create(Node.Map);	420	const node = try self.allocator.create(Node.Map);
365	errdefer self.allocator.destroy(node);	421	errdefer self.allocator.destroy(node);
366	node.* = .{ .start = start };	422	node.* = .{};
367	node.base.tree = self.tree;	423	node.base.tree = self.tree;
		424	node.base.start = self.token_it.pos;
		425	errdefer {
		426	for (node.values.items) \|entry\| {
		427	if (entry.value) \|val\| {
		428	val.deinit(self.allocator);
		429	self.allocator.destroy(val);
		430	}
		431	}
		432	node.values.deinit(self.allocator);
		433	}
368		434
369	self.token_it.seekTo(start);	435	log.debug("(map) begin {s}@{d}", .{ @tagName(self.tree.tokens[node.base.start].id), node.base.start });
370
371	log.debug("Map start: {}, {}", .{ start, self.tree.tokens[start] });
372		436
373	const col = self.getCol(start);	437	const col = self.getCol(node.base.start);
374		438
375	while (true) {	439	while (true) {
376	self.eatCommentsAndSpace();	440	self.eatCommentsAndSpace(&.{});
377		441
378	// Parse key.	442	// Parse key
379	const key_pos = self.token_it.pos;	443	const key_pos = self.token_it.pos;
380	if (self.getCol(key_pos) != col) {	444	if (self.getCol(key_pos) < col) {
381	break;	445	break;
382	}	446	}
383		447
384	const key = self.token_it.next();	448	const key = self.token_it.next() orelse return error.UnexpectedEof;
385	switch (key.id) {	449	switch (key.id) {
386	.Literal => {},	450	.literal => {},
387	else => {	451	.doc_start, .doc_end, .eof => {
388	self.token_it.seekBy(-1);	452	self.token_it.seekBy(-1);
389	break;	453	break;
390	},	454	},
		455	else => {
		456	// TODO key not being a literal
		457	return error.Unhandled;
		458	},
391	}	459	}
392		460
393	log.debug("Map key: {}, '{s}'", .{ key, self.tree.source[key.start..key.end] });	461	log.debug("(map) key {s}@{d}", .{ self.tree.getRaw(key_pos, key_pos), key_pos });
394		462
395	// Separator	463	// Separator
396	_ = try self.expectToken(.MapValueInd);	464	_ = try self.expectToken(.map_value_ind, &.{ .new_line, .comment });
397		465
398	// Parse value.	466	// Parse value
399	const value: *Node = value: {	467	const val = try self.value();
400	if (self.eatToken(.NewLine)) \|_\| {	468	errdefer if (val) \|v\| {
401	self.eatCommentsAndSpace();	469	v.deinit(self.allocator);
402		470	self.allocator.destroy(v);
403	// Explicit, complex value such as list or map.	471	};
404	const value_pos = self.token_it.pos;	472
405	const value = self.token_it.next();	473	if (val) \|v\| {
406	switch (value.id) {	474	if (self.getCol(v.start) < self.getCol(key_pos)) {
407	.Literal, .SingleQuote, .DoubleQuote => {	475	return error.MalformedYaml;
408	// Assume nested map.	476	}
409	const map_node = try self.map(value_pos);	477	if (v.cast(Node.Value)) \|_\| {
410	break :value &map_node.base;	478	if (self.getCol(v.start) == self.getCol(key_pos)) {
411	},	479	return error.MalformedYaml;
412	.SeqItemInd => {
413	// Assume list of values.
414	const list_node = try self.list(value_pos);
415	break :value &list_node.base;
416	},
417	else => {
418	log.err("{}", .{key});
419	return error.Unhandled;
420	},
421	}
422	} else {
423	self.eatCommentsAndSpace();
424
425	const value_pos = self.token_it.pos;
426	const value = self.token_it.next();
427	switch (value.id) {
428	.Literal, .SingleQuote, .DoubleQuote => {
429	// Assume leaf value.
430	const leaf_node = try self.leaf_value(value_pos);
431	break :value &leaf_node.base;
432	},
433	.FlowSeqStart => {
434	const list_node = try self.list_bracketed(value_pos);
435	break :value &list_node.base;
436	},
437	else => {
438	log.err("{}", .{key});
439	return error.Unhandled;
440	},
441	}	480	}
442	}	481	}
443	};	482	}
444	log.debug("Map value: {}", .{value});
445		483
446	try node.values.append(self.allocator, .{	484	try node.values.append(self.allocator, .{
447	.key = key_pos,	485	.key = key_pos,
448	.value = value,	486	.value = val,
449	});	487	});
450
451	_ = self.eatToken(.NewLine);
452	}	488	}
453		489
454	node.end = self.token_it.pos - 1;	490	node.base.end = self.token_it.pos - 1;
455		491
456	log.debug("Map end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] });	492	log.debug("(map) end {s}@{d}", .{ @tagName(self.tree.tokens[node.base.end].id), node.base.end });
457		493
458	return node;	494	return &node.base;
459	}	495	}
460		496
461	fn list(self: Parser, start: TokenIndex) ParseError!Node.List {	497	fn list(self: Parser) ParseError!Node {
462	const node = try self.allocator.create(Node.List);	498	const node = try self.allocator.create(Node.List);
463	errdefer self.allocator.destroy(node);	499	errdefer self.allocator.destroy(node);
464	node.* = .{	500	node.* = .{};
465	.start = start,
466	};
467	node.base.tree = self.tree;	501	node.base.tree = self.tree;
		502	node.base.start = self.token_it.pos;
		503	errdefer {
		504	for (node.values.items) \|val\| {
		505	val.deinit(self.allocator);
		506	self.allocator.destroy(val);
		507	}
		508	node.values.deinit(self.allocator);
		509	}
468		510
469	self.token_it.seekTo(start);	511	log.debug("(list) begin {s}@{d}", .{ @tagName(self.tree.tokens[node.base.start].id), node.base.start });
470
471	log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] });
472
473	const col = self.getCol(start);
474		512
475	while (true) {	513	while (true) {
476	self.eatCommentsAndSpace();	514	self.eatCommentsAndSpace(&.{});
477		515
478	if (self.getCol(self.token_it.pos) != col) {	516	_ = self.eatToken(.seq_item_ind, &.{}) orelse break;
479	break;
480	}
481	_ = self.eatToken(.SeqItemInd) orelse {
482	break;
483	};
484		517
485	const pos = self.token_it.pos;	518	const val = (try self.value()) orelse return error.MalformedYaml;
486	const token = self.token_it.next();	519	try node.values.append(self.allocator, val);
487	const value: *Node = value: {
488	switch (token.id) {
489	.Literal, .SingleQuote, .DoubleQuote => {
490	if (self.eatToken(.MapValueInd)) \|_\| {
491	// nested map
492	const map_node = try self.map(pos);
493	break :value &map_node.base;
494	} else {
495	// standalone (leaf) value
496	const leaf_node = try self.leaf_value(pos);
497	break :value &leaf_node.base;
498	}
499	},
500	.FlowSeqStart => {
501	const list_node = try self.list_bracketed(pos);
502	break :value &list_node.base;
503	},
504	else => {
505	log.err("{}", .{token});
506	return error.Unhandled;
507	},
508	}
509	};
510	try node.values.append(self.allocator, value);
511
512	_ = self.eatToken(.NewLine);
513	}	520	}
514		521
515	node.end = self.token_it.pos - 1;	522	node.base.end = self.token_it.pos - 1;
516		523
517	log.debug("List end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] });	524	log.debug("(list) end {s}@{d}", .{ @tagName(self.tree.tokens[node.base.end].id), node.base.end });
518		525
519	return node;	526	return &node.base;
520	}	527	}
521		528
522	fn list_bracketed(self: Parser, start: TokenIndex) ParseError!Node.List {	529	fn list_bracketed(self: Parser) ParseError!Node {
523	const node = try self.allocator.create(Node.List);	530	const node = try self.allocator.create(Node.List);
524	errdefer self.allocator.destroy(node);	531	errdefer self.allocator.destroy(node);
525	node.* = .{ .start = start };	532	node.* = .{};
526	node.base.tree = self.tree;	533	node.base.tree = self.tree;
		534	node.base.start = self.token_it.pos;
		535	errdefer {
		536	for (node.values.items) \|val\| {
		537	val.deinit(self.allocator);
		538	self.allocator.destroy(val);
		539	}
		540	node.values.deinit(self.allocator);
		541	}
527		542
528	self.token_it.seekTo(start);	543	log.debug("(list) begin {s}@{d}", .{ @tagName(self.tree.tokens[node.base.start].id), node.base.start });
529
530	log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] });
531		544
532	_ = try self.expectToken(.FlowSeqStart);	545	_ = try self.expectToken(.flow_seq_start, &.{});
533		546
534	while (true) {	547	while (true) {
535	_ = self.eatToken(.NewLine);	548	self.eatCommentsAndSpace(&.{.comment});
536	self.eatCommentsAndSpace();
537		549
538	const pos = self.token_it.pos;	550	if (self.eatToken(.flow_seq_end, &.{.comment})) \|pos\| {
539	const token = self.token_it.next();	551	node.base.end = pos;
540		552	break;
541	log.debug("Next token: {}, {}", .{ pos, token });	553	}
		554	_ = self.eatToken(.comma, &.{.comment});
542		555
543	const value: *Node = value: {	556	const val = (try self.value()) orelse return error.MalformedYaml;
544	switch (token.id) {	557	try node.values.append(self.allocator, val);
545	.FlowSeqStart => {
546	const list_node = try self.list_bracketed(pos);
547	break :value &list_node.base;
548	},
549	.FlowSeqEnd => {
550	break;
551	},
552	.Literal, .SingleQuote, .DoubleQuote => {
553	const leaf_node = try self.leaf_value(pos);
554	_ = self.eatToken(.Comma);
555	// TODO newline
556	break :value &leaf_node.base;
557	},
558	else => {
559	log.err("{}", .{token});
560	return error.Unhandled;
561	},
562	}
563	};
564	try node.values.append(self.allocator, value);
565	}	558	}
566		559
567	node.end = self.token_it.pos - 1;	560	log.debug("(list) end {s}@{d}", .{ @tagName(self.tree.tokens[node.base.end].id), node.base.end });
568
569	log.debug("List end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] });
570		561
571	return node;	562	return &node.base;
572	}	563	}
573		564
574	fn leaf_value(self: Parser, start: TokenIndex) ParseError!Node.Value {	565	fn leaf_value(self: Parser) ParseError!Node {
575	const node = try self.allocator.create(Node.Value);	566	const node = try self.allocator.create(Node.Value);
576	errdefer self.allocator.destroy(node);	567	errdefer self.allocator.destroy(node);
577	node.* = .{ .start = start };	568	node.* = .{ .string_value = .{} };
578	node.base.tree = self.tree;	569	node.base.tree = self.tree;
579		570	node.base.start = self.token_it.pos;
580	self.token_it.seekTo(start);	571	errdefer node.string_value.deinit(self.allocator);
581		572
582	log.debug("Leaf start: {}, {}", .{ node.start.?, self.tree.tokens[node.start.?] });	573	// TODO handle multiline strings in new block scope
583		574	while (self.token_it.next()) \|tok\| {
584	parse: {	575	switch (tok.id) {
585	if (self.eatToken(.SingleQuote)) \|_\| {	576	.single_quoted => {
586	node.start = node.start.? + 1;	577	node.base.end = self.token_it.pos - 1;
587	while (true) {	578	const raw = self.tree.getRaw(node.base.start, node.base.end);
588	const tok = self.token_it.next();	579	try self.parseSingleQuoted(node, raw);
589	switch (tok.id) {	580	break;
590	.SingleQuote => {	581	},
591	node.end = self.token_it.pos - 2;	582	.double_quoted => {
592	break :parse;	583	node.base.end = self.token_it.pos - 1;
593	},	584	const raw = self.tree.getRaw(node.base.start, node.base.end);
594	.NewLine => return error.UnexpectedToken,	585	try self.parseDoubleQuoted(node, raw);
595	else => {},	586	break;
596	}	587	},
597	}	588	.literal => {},
598	}	589	.space => {
599		590	const trailing = self.token_it.pos - 2;
600	if (self.eatToken(.DoubleQuote)) \|_\| {	591	self.eatCommentsAndSpace(&.{});
601	node.start = node.start.? + 1;	592	if (self.token_it.peek()) \|peek\| {
602	while (true) {	593	if (peek.id != .literal) {
603	const tok = self.token_it.next();	594	node.base.end = trailing;
604	switch (tok.id) {	595	const raw = self.tree.getRaw(node.base.start, node.base.end);
605	.DoubleQuote => {	596	try node.string_value.appendSlice(self.allocator, raw);
606	node.end = self.token_it.pos - 2;	597	break;
607	break :parse;
608	},
609	.NewLine => return error.UnexpectedToken,
610	else => {},
611	}
612	}
613	}
614
615	// TODO handle multiline strings in new block scope
616	while (true) {
617	const tok = self.token_it.next();
618	switch (tok.id) {
619	.Literal => {},
620	.Space => {
621	const trailing = self.token_it.pos - 2;
622	self.eatCommentsAndSpace();
623	if (self.token_it.peek()) \|peek\| {
624	if (peek.id != .Literal) {
625	node.end = trailing;
626	break;
627	}
628	}	598	}
629	},	599	}
630	else => {	600	},
631	self.token_it.seekBy(-1);	601	else => {
632	node.end = self.token_it.pos - 1;	602	self.token_it.seekBy(-1);
633	break;	603	node.base.end = self.token_it.pos - 1;
634	},	604	const raw = self.tree.getRaw(node.base.start, node.base.end);
635	}	605	try node.string_value.appendSlice(self.allocator, raw);
		606	break;
		607	},
636	}	608	}
637	}	609	}
638		610
639	log.debug("Leaf end: {}, {}", .{ node.end.?, self.tree.tokens[node.end.?] });	611	log.debug("(leaf) {s}", .{self.tree.getRaw(node.base.start, node.base.end)});
640		612
641	return node;	613	return &node.base;
642	}	614	}
643		615
644	fn eatCommentsAndSpace(self: *Parser) void {	616	fn eatCommentsAndSpace(self: *Parser, comptime exclusions: []const Token.Id) void {
645	while (true) {	617	log.debug("eatCommentsAndSpace", .{});
646	_ = self.token_it.peek() orelse return;	618	outer: while (self.token_it.next()) \|token\| {
647	const token = self.token_it.next();	619	log.debug(" (token '{s}')", .{@tagName(token.id)});
648	switch (token.id) {	620	switch (token.id) {
649	.Comment, .Space => {},	621	.comment, .space, .new_line => \|space\| {
		622	inline for (exclusions) \|excl\| {
		623	if (excl == space) {
		624	self.token_it.seekBy(-1);
		625	break :outer;
		626	}
		627	} else continue;
		628	},
650	else => {	629	else => {
651	self.token_it.seekBy(-1);	630	self.token_it.seekBy(-1);
652	break;	631	break;
...	@@ -655,25 +634,24 @@ const Parser = struct {	...	@@ -655,25 +634,24 @@ const Parser = struct {
655	}	634	}
656	}	635	}
657		636
658	fn eatToken(self: *Parser, id: Token.Id) ?TokenIndex {	637	fn eatToken(self: *Parser, id: Token.Id, comptime exclusions: []const Token.Id) ?TokenIndex {
659	while (true) {	638	log.debug("eatToken('{s}')", .{@tagName(id)});
660	const pos = self.token_it.pos;	639	self.eatCommentsAndSpace(exclusions);
661	_ = self.token_it.peek() orelse return null;	640	const pos = self.token_it.pos;
662	const token = self.token_it.next();	641	const token = self.token_it.next() orelse return null;
663	switch (token.id) {	642	if (token.id == id) {
664	.Comment, .Space => continue,	643	log.debug(" (found at {d})", .{pos});
665	else => \|next_id\| if (next_id == id) {	644	return pos;
666	return pos;	645	} else {
667	} else {	646	log.debug(" (not found)", .{});
668	self.token_it.seekTo(pos);	647	self.token_it.seekBy(-1);
669	return null;	648	return null;
670	},
671	}
672	}	649	}
673	}	650	}
674		651
675	fn expectToken(self: *Parser, id: Token.Id) ParseError!TokenIndex {	652	fn expectToken(self: *Parser, id: Token.Id, comptime exclusions: []const Token.Id) ParseError!TokenIndex {
676	return self.eatToken(id) orelse error.UnexpectedToken;	653	log.debug("expectToken('{s}')", .{@tagName(id)});
		654	return self.eatToken(id, exclusions) orelse error.UnexpectedToken;
677	}	655	}
678		656
679	fn getLine(self: *Parser, index: TokenIndex) usize {	657	fn getLine(self: *Parser, index: TokenIndex) usize {
...	@@ -683,8 +661,85 @@ const Parser = struct {	...	@@ -683,8 +661,85 @@ const Parser = struct {
683	fn getCol(self: *Parser, index: TokenIndex) usize {	661	fn getCol(self: *Parser, index: TokenIndex) usize {
684	return self.line_cols.get(index).?.col;	662	return self.line_cols.get(index).?.col;
685	}	663	}
		664
		665	fn parseSingleQuoted(self: Parser, node: Node.Value, raw: []const u8) ParseError!void {
		666	assert(raw[0] == '\'' and raw[raw.len - 1] == '\'');
		667
		668	const raw_no_quotes = raw[1 .. raw.len - 1];
		669	try node.string_value.ensureTotalCapacity(self.allocator, raw_no_quotes.len);
		670
		671	var state: enum {
		672	start,
		673	escape,
		674	} = .start;
		675	var index: usize = 0;
		676
		677	while (index < raw_no_quotes.len) : (index += 1) {
		678	const c = raw_no_quotes[index];
		679	switch (state) {
		680	.start => switch (c) {
		681	'\'' => {
		682	state = .escape;
		683	},
		684	else => {
		685	node.string_value.appendAssumeCapacity(c);
		686	},
		687	},
		688	.escape => switch (c) {
		689	'\'' => {
		690	state = .start;
		691	node.string_value.appendAssumeCapacity(c);
		692	},
		693	else => return error.InvalidEscapeSequence,
		694	},
		695	}
		696	}
		697	}
		698
		699	fn parseDoubleQuoted(self: Parser, node: Node.Value, raw: []const u8) ParseError!void {
		700	assert(raw[0] == '"' and raw[raw.len - 1] == '"');
		701
		702	const raw_no_quotes = raw[1 .. raw.len - 1];
		703	try node.string_value.ensureTotalCapacity(self.allocator, raw_no_quotes.len);
		704
		705	var state: enum {
		706	start,
		707	escape,
		708	} = .start;
		709
		710	var index: usize = 0;
		711	while (index < raw_no_quotes.len) : (index += 1) {
		712	const c = raw_no_quotes[index];
		713	switch (state) {
		714	.start => switch (c) {
		715	'\\' => {
		716	state = .escape;
		717	},
		718	else => {
		719	node.string_value.appendAssumeCapacity(c);
		720	},
		721	},
		722	.escape => switch (c) {
		723	'n' => {
		724	state = .start;
		725	node.string_value.appendAssumeCapacity('\n');
		726	},
		727	't' => {
		728	state = .start;
		729	node.string_value.appendAssumeCapacity('\t');
		730	},
		731	'"' => {
		732	state = .start;
		733	node.string_value.appendAssumeCapacity('"');
		734	},
		735	else => return error.InvalidEscapeSequence,
		736	},
		737	}
		738	}
		739	}
686	};	740	};
687		741
688	test {	742	test {
		743	std.testing.refAllDecls(@This());
689	_ = @import("parse/test.zig");	744	_ = @import("parse/test.zig");
690	}	745	}

src/link/tapi/parse/test.zig+385-179

...	@@ -21,45 +21,45 @@ test "explicit doc" {	...	@@ -21,45 +21,45 @@ test "explicit doc" {
21	try testing.expectEqual(tree.docs.items.len, 1);	21	try testing.expectEqual(tree.docs.items.len, 1);
22		22
23	const doc = tree.docs.items[0].cast(Node.Doc).?;	23	const doc = tree.docs.items[0].cast(Node.Doc).?;
24	try testing.expectEqual(doc.start.?, 0);	24	try testing.expectEqual(doc.base.start, 0);
25	try testing.expectEqual(doc.end.?, tree.tokens.len - 2);	25	try testing.expectEqual(doc.base.end, tree.tokens.len - 2);
26		26
27	const directive = tree.tokens[doc.directive.?];	27	const directive = tree.tokens[doc.directive.?];
28	try testing.expectEqual(directive.id, .Literal);	28	try testing.expectEqual(directive.id, .literal);
29	try testing.expect(mem.eql(u8, "tapi-tbd", tree.source[directive.start..directive.end]));	29	try testing.expectEqualStrings("tapi-tbd", tree.source[directive.start..directive.end]);
30		30
31	try testing.expect(doc.value != null);	31	try testing.expect(doc.value != null);
32	try testing.expectEqual(doc.value.?.tag, .map);	32	try testing.expectEqual(doc.value.?.tag, .map);
33		33
34	const map = doc.value.?.cast(Node.Map).?;	34	const map = doc.value.?.cast(Node.Map).?;
35	try testing.expectEqual(map.start.?, 5);	35	try testing.expectEqual(map.base.start, 5);
36	try testing.expectEqual(map.end.?, 14);	36	try testing.expectEqual(map.base.end, 14);
37	try testing.expectEqual(map.values.items.len, 2);	37	try testing.expectEqual(map.values.items.len, 2);
38		38
39	{	39	{
40	const entry = map.values.items[0];	40	const entry = map.values.items[0];
41		41
42	const key = tree.tokens[entry.key];	42	const key = tree.tokens[entry.key];
43	try testing.expectEqual(key.id, .Literal);	43	try testing.expectEqual(key.id, .literal);
44	try testing.expect(mem.eql(u8, "tbd-version", tree.source[key.start..key.end]));	44	try testing.expectEqualStrings("tbd-version", tree.source[key.start..key.end]);
45		45
46	const value = entry.value.cast(Node.Value).?;	46	const value = entry.value.?.cast(Node.Value).?;
47	const value_tok = tree.tokens[value.start.?];	47	const value_tok = tree.tokens[value.base.start];
48	try testing.expectEqual(value_tok.id, .Literal);	48	try testing.expectEqual(value_tok.id, .literal);
49	try testing.expect(mem.eql(u8, "4", tree.source[value_tok.start..value_tok.end]));	49	try testing.expectEqualStrings("4", tree.source[value_tok.start..value_tok.end]);
50	}	50	}
51		51
52	{	52	{
53	const entry = map.values.items[1];	53	const entry = map.values.items[1];
54		54
55	const key = tree.tokens[entry.key];	55	const key = tree.tokens[entry.key];
56	try testing.expectEqual(key.id, .Literal);	56	try testing.expectEqual(key.id, .literal);
57	try testing.expect(mem.eql(u8, "abc-version", tree.source[key.start..key.end]));	57	try testing.expectEqualStrings("abc-version", tree.source[key.start..key.end]);
58		58
59	const value = entry.value.cast(Node.Value).?;	59	const value = entry.value.?.cast(Node.Value).?;
60	const value_tok = tree.tokens[value.start.?];	60	const value_tok = tree.tokens[value.base.start];
61	try testing.expectEqual(value_tok.id, .Literal);	61	try testing.expectEqual(value_tok.id, .literal);
62	try testing.expect(mem.eql(u8, "5", tree.source[value_tok.start..value_tok.end]));	62	try testing.expectEqualStrings("5", tree.source[value_tok.start..value_tok.end]);
63	}	63	}
64	}	64	}
65		65
...	@@ -77,39 +77,31 @@ test "leaf in quotes" {	...	@@ -77,39 +77,31 @@ test "leaf in quotes" {
77	try testing.expectEqual(tree.docs.items.len, 1);	77	try testing.expectEqual(tree.docs.items.len, 1);
78		78
79	const doc = tree.docs.items[0].cast(Node.Doc).?;	79	const doc = tree.docs.items[0].cast(Node.Doc).?;
80	try testing.expectEqual(doc.start.?, 0);	80	try testing.expectEqual(doc.base.start, 0);
81	try testing.expectEqual(doc.end.?, tree.tokens.len - 2);	81	try testing.expectEqual(doc.base.end, tree.tokens.len - 2);
82	try testing.expect(doc.directive == null);	82	try testing.expect(doc.directive == null);
83		83
84	try testing.expect(doc.value != null);	84	try testing.expect(doc.value != null);
85	try testing.expectEqual(doc.value.?.tag, .map);	85	try testing.expectEqual(doc.value.?.tag, .map);
86		86
87	const map = doc.value.?.cast(Node.Map).?;	87	const map = doc.value.?.cast(Node.Map).?;
88	try testing.expectEqual(map.start.?, 0);	88	try testing.expectEqual(map.base.start, 0);
89	try testing.expectEqual(map.end.?, tree.tokens.len - 2);	89	try testing.expectEqual(map.base.end, tree.tokens.len - 2);
90	try testing.expectEqual(map.values.items.len, 3);	90	try testing.expectEqual(map.values.items.len, 3);
91		91
92	{	92	{
93	const entry = map.values.items[0];	93	const entry = map.values.items[0];
94		94
95	const key = tree.tokens[entry.key];	95	const key = tree.tokens[entry.key];
96	try testing.expectEqual(key.id, .Literal);	96	try testing.expectEqual(key.id, .literal);
97	try testing.expect(mem.eql(	97	try testing.expectEqualStrings("key1", tree.source[key.start..key.end]);
98	u8,	98
99	"key1",	99	const value = entry.value.?.cast(Node.Value).?;
100	tree.source[key.start..key.end],	100	const start = tree.tokens[value.base.start];
101	));	101	const end = tree.tokens[value.base.end];
102		102	try testing.expectEqual(start.id, .literal);
103	const value = entry.value.cast(Node.Value).?;	103	try testing.expectEqual(end.id, .literal);
104	const start = tree.tokens[value.start.?];	104	try testing.expectEqualStrings("no quotes", tree.source[start.start..end.end]);
105	const end = tree.tokens[value.end.?];
106	try testing.expectEqual(start.id, .Literal);
107	try testing.expectEqual(end.id, .Literal);
108	try testing.expect(mem.eql(
109	u8,
110	"no quotes",
111	tree.source[start.start..end.end],
112	));
113	}	105	}
114	}	106	}
115		107
...	@@ -128,70 +120,60 @@ test "nested maps" {	...	@@ -128,70 +120,60 @@ test "nested maps" {
128	try testing.expectEqual(tree.docs.items.len, 1);	120	try testing.expectEqual(tree.docs.items.len, 1);
129		121
130	const doc = tree.docs.items[0].cast(Node.Doc).?;	122	const doc = tree.docs.items[0].cast(Node.Doc).?;
131	try testing.expectEqual(doc.start.?, 0);	123	try testing.expectEqual(doc.base.start, 0);
132	try testing.expectEqual(doc.end.?, tree.tokens.len - 2);	124	try testing.expectEqual(doc.base.end, tree.tokens.len - 2);
133	try testing.expect(doc.directive == null);	125	try testing.expect(doc.directive == null);
134		126
135	try testing.expect(doc.value != null);	127	try testing.expect(doc.value != null);
136	try testing.expectEqual(doc.value.?.tag, .map);	128	try testing.expectEqual(doc.value.?.tag, .map);
137		129
138	const map = doc.value.?.cast(Node.Map).?;	130	const map = doc.value.?.cast(Node.Map).?;
139	try testing.expectEqual(map.start.?, 0);	131	try testing.expectEqual(map.base.start, 0);
140	try testing.expectEqual(map.end.?, tree.tokens.len - 2);	132	try testing.expectEqual(map.base.end, tree.tokens.len - 2);
141	try testing.expectEqual(map.values.items.len, 2);	133	try testing.expectEqual(map.values.items.len, 2);
142		134
143	{	135	{
144	const entry = map.values.items[0];	136	const entry = map.values.items[0];
145		137
146	const key = tree.tokens[entry.key];	138	const key = tree.tokens[entry.key];
147	try testing.expectEqual(key.id, .Literal);	139	try testing.expectEqual(key.id, .literal);
148	try testing.expect(mem.eql(u8, "key1", tree.source[key.start..key.end]));	140	try testing.expectEqualStrings("key1", tree.source[key.start..key.end]);
149		141
150	const nested_map = entry.value.cast(Node.Map).?;	142	const nested_map = entry.value.?.cast(Node.Map).?;
151	try testing.expectEqual(nested_map.start.?, 4);	143	try testing.expectEqual(nested_map.base.start, 4);
152	try testing.expectEqual(nested_map.end.?, 16);	144	try testing.expectEqual(nested_map.base.end, 16);
153	try testing.expectEqual(nested_map.values.items.len, 2);	145	try testing.expectEqual(nested_map.values.items.len, 2);
154		146
155	{	147	{
156	const nested_entry = nested_map.values.items[0];	148	const nested_entry = nested_map.values.items[0];
157		149
158	const nested_key = tree.tokens[nested_entry.key];	150	const nested_key = tree.tokens[nested_entry.key];
159	try testing.expectEqual(nested_key.id, .Literal);	151	try testing.expectEqual(nested_key.id, .literal);
160	try testing.expect(mem.eql(	152	try testing.expectEqualStrings("key1_1", tree.source[nested_key.start..nested_key.end]);
161	u8,	153
162	"key1_1",	154	const nested_value = nested_entry.value.?.cast(Node.Value).?;
163	tree.source[nested_key.start..nested_key.end],	155	const nested_value_tok = tree.tokens[nested_value.base.start];
164	));	156	try testing.expectEqual(nested_value_tok.id, .literal);
165		157	try testing.expectEqualStrings(
166	const nested_value = nested_entry.value.cast(Node.Value).?;
167	const nested_value_tok = tree.tokens[nested_value.start.?];
168	try testing.expectEqual(nested_value_tok.id, .Literal);
169	try testing.expect(mem.eql(
170	u8,
171	"value1_1",	158	"value1_1",
172	tree.source[nested_value_tok.start..nested_value_tok.end],	159	tree.source[nested_value_tok.start..nested_value_tok.end],
173	));	160	);
174	}	161	}
175		162
176	{	163	{
177	const nested_entry = nested_map.values.items[1];	164	const nested_entry = nested_map.values.items[1];
178		165
179	const nested_key = tree.tokens[nested_entry.key];	166	const nested_key = tree.tokens[nested_entry.key];
180	try testing.expectEqual(nested_key.id, .Literal);	167	try testing.expectEqual(nested_key.id, .literal);
181	try testing.expect(mem.eql(	168	try testing.expectEqualStrings("key1_2", tree.source[nested_key.start..nested_key.end]);
182	u8,	169
183	"key1_2",	170	const nested_value = nested_entry.value.?.cast(Node.Value).?;
184	tree.source[nested_key.start..nested_key.end],	171	const nested_value_tok = tree.tokens[nested_value.base.start];
185	));	172	try testing.expectEqual(nested_value_tok.id, .literal);
186		173	try testing.expectEqualStrings(
187	const nested_value = nested_entry.value.cast(Node.Value).?;
188	const nested_value_tok = tree.tokens[nested_value.start.?];
189	try testing.expectEqual(nested_value_tok.id, .Literal);
190	try testing.expect(mem.eql(
191	u8,
192	"value1_2",	174	"value1_2",
193	tree.source[nested_value_tok.start..nested_value_tok.end],	175	tree.source[nested_value_tok.start..nested_value_tok.end],
194	));	176	);
195	}	177	}
196	}	178	}
197		179
...	@@ -199,17 +181,13 @@ test "nested maps" {	...	@@ -199,17 +181,13 @@ test "nested maps" {
199	const entry = map.values.items[1];	181	const entry = map.values.items[1];
200		182
201	const key = tree.tokens[entry.key];	183	const key = tree.tokens[entry.key];
202	try testing.expectEqual(key.id, .Literal);	184	try testing.expectEqual(key.id, .literal);
203	try testing.expect(mem.eql(u8, "key2", tree.source[key.start..key.end]));	185	try testing.expectEqualStrings("key2", tree.source[key.start..key.end]);
204		186
205	const value = entry.value.cast(Node.Value).?;	187	const value = entry.value.?.cast(Node.Value).?;
206	const value_tok = tree.tokens[value.start.?];	188	const value_tok = tree.tokens[value.base.start];
207	try testing.expectEqual(value_tok.id, .Literal);	189	try testing.expectEqual(value_tok.id, .literal);
208	try testing.expect(mem.eql(	190	try testing.expectEqualStrings("value2", tree.source[value_tok.start..value_tok.end]);
209	u8,
210	"value2",
211	tree.source[value_tok.start..value_tok.end],
212	));
213	}	191	}
214	}	192	}
215		193
...	@@ -227,46 +205,46 @@ test "map of list of values" {	...	@@ -227,46 +205,46 @@ test "map of list of values" {
227	try testing.expectEqual(tree.docs.items.len, 1);	205	try testing.expectEqual(tree.docs.items.len, 1);
228		206
229	const doc = tree.docs.items[0].cast(Node.Doc).?;	207	const doc = tree.docs.items[0].cast(Node.Doc).?;
230	try testing.expectEqual(doc.start.?, 0);	208	try testing.expectEqual(doc.base.start, 0);
231	try testing.expectEqual(doc.end.?, tree.tokens.len - 2);	209	try testing.expectEqual(doc.base.end, tree.tokens.len - 2);
232		210
233	try testing.expect(doc.value != null);	211	try testing.expect(doc.value != null);
234	try testing.expectEqual(doc.value.?.tag, .map);	212	try testing.expectEqual(doc.value.?.tag, .map);
235		213
236	const map = doc.value.?.cast(Node.Map).?;	214	const map = doc.value.?.cast(Node.Map).?;
237	try testing.expectEqual(map.start.?, 0);	215	try testing.expectEqual(map.base.start, 0);
238	try testing.expectEqual(map.end.?, tree.tokens.len - 2);	216	try testing.expectEqual(map.base.end, tree.tokens.len - 2);
239	try testing.expectEqual(map.values.items.len, 1);	217	try testing.expectEqual(map.values.items.len, 1);
240		218
241	const entry = map.values.items[0];	219	const entry = map.values.items[0];
242	const key = tree.tokens[entry.key];	220	const key = tree.tokens[entry.key];
243	try testing.expectEqual(key.id, .Literal);	221	try testing.expectEqual(key.id, .literal);
244	try testing.expect(mem.eql(u8, "ints", tree.source[key.start..key.end]));	222	try testing.expectEqualStrings("ints", tree.source[key.start..key.end]);
245		223
246	const value = entry.value.cast(Node.List).?;	224	const value = entry.value.?.cast(Node.List).?;
247	try testing.expectEqual(value.start.?, 4);	225	try testing.expectEqual(value.base.start, 4);
248	try testing.expectEqual(value.end.?, tree.tokens.len - 2);	226	try testing.expectEqual(value.base.end, tree.tokens.len - 2);
249	try testing.expectEqual(value.values.items.len, 3);	227	try testing.expectEqual(value.values.items.len, 3);
250		228
251	{	229	{
252	const elem = value.values.items[0].cast(Node.Value).?;	230	const elem = value.values.items[0].cast(Node.Value).?;
253	const leaf = tree.tokens[elem.start.?];	231	const leaf = tree.tokens[elem.base.start];
254	try testing.expectEqual(leaf.id, .Literal);	232	try testing.expectEqual(leaf.id, .literal);
255	try testing.expect(mem.eql(u8, "0", tree.source[leaf.start..leaf.end]));	233	try testing.expectEqualStrings("0", tree.source[leaf.start..leaf.end]);
256	}	234	}
257		235
258	{	236	{
259	const elem = value.values.items[1].cast(Node.Value).?;	237	const elem = value.values.items[1].cast(Node.Value).?;
260	const leaf = tree.tokens[elem.start.?];	238	const leaf = tree.tokens[elem.base.start];
261	try testing.expectEqual(leaf.id, .Literal);	239	try testing.expectEqual(leaf.id, .literal);
262	try testing.expect(mem.eql(u8, "1", tree.source[leaf.start..leaf.end]));	240	try testing.expectEqualStrings("1", tree.source[leaf.start..leaf.end]);
263	}	241	}
264		242
265	{	243	{
266	const elem = value.values.items[2].cast(Node.Value).?;	244	const elem = value.values.items[2].cast(Node.Value).?;
267	const leaf = tree.tokens[elem.start.?];	245	const leaf = tree.tokens[elem.base.start];
268	try testing.expectEqual(leaf.id, .Literal);	246	try testing.expectEqual(leaf.id, .literal);
269	try testing.expect(mem.eql(u8, "2", tree.source[leaf.start..leaf.end]));	247	try testing.expectEqualStrings("2", tree.source[leaf.start..leaf.end]);
270	}	248	}
271	}	249	}
272		250
...	@@ -285,64 +263,64 @@ test "map of list of maps" {	...	@@ -285,64 +263,64 @@ test "map of list of maps" {
285	try testing.expectEqual(tree.docs.items.len, 1);	263	try testing.expectEqual(tree.docs.items.len, 1);
286		264
287	const doc = tree.docs.items[0].cast(Node.Doc).?;	265	const doc = tree.docs.items[0].cast(Node.Doc).?;
288	try testing.expectEqual(doc.start.?, 0);	266	try testing.expectEqual(doc.base.start, 0);
289	try testing.expectEqual(doc.end.?, tree.tokens.len - 2);	267	try testing.expectEqual(doc.base.end, tree.tokens.len - 2);
290		268
291	try testing.expect(doc.value != null);	269	try testing.expect(doc.value != null);
292	try testing.expectEqual(doc.value.?.tag, .map);	270	try testing.expectEqual(doc.value.?.tag, .map);
293		271
294	const map = doc.value.?.cast(Node.Map).?;	272	const map = doc.value.?.cast(Node.Map).?;
295	try testing.expectEqual(map.start.?, 0);	273	try testing.expectEqual(map.base.start, 0);
296	try testing.expectEqual(map.end.?, tree.tokens.len - 2);	274	try testing.expectEqual(map.base.end, tree.tokens.len - 2);
297	try testing.expectEqual(map.values.items.len, 1);	275	try testing.expectEqual(map.values.items.len, 1);
298		276
299	const entry = map.values.items[0];	277	const entry = map.values.items[0];
300	const key = tree.tokens[entry.key];	278	const key = tree.tokens[entry.key];
301	try testing.expectEqual(key.id, .Literal);	279	try testing.expectEqual(key.id, .literal);
302	try testing.expect(mem.eql(u8, "key1", tree.source[key.start..key.end]));	280	try testing.expectEqualStrings("key1", tree.source[key.start..key.end]);
303		281
304	const value = entry.value.cast(Node.List).?;	282	const value = entry.value.?.cast(Node.List).?;
305	try testing.expectEqual(value.start.?, 3);	283	try testing.expectEqual(value.base.start, 3);
306	try testing.expectEqual(value.end.?, tree.tokens.len - 2);	284	try testing.expectEqual(value.base.end, tree.tokens.len - 2);
307	try testing.expectEqual(value.values.items.len, 3);	285	try testing.expectEqual(value.values.items.len, 3);
308		286
309	{	287	{
310	const elem = value.values.items[0].cast(Node.Map).?;	288	const elem = value.values.items[0].cast(Node.Map).?;
311	const nested = elem.values.items[0];	289	const nested = elem.values.items[0];
312	const nested_key = tree.tokens[nested.key];	290	const nested_key = tree.tokens[nested.key];
313	try testing.expectEqual(nested_key.id, .Literal);	291	try testing.expectEqual(nested_key.id, .literal);
314	try testing.expect(mem.eql(u8, "key2", tree.source[nested_key.start..nested_key.end]));	292	try testing.expectEqualStrings("key2", tree.source[nested_key.start..nested_key.end]);
315		293
316	const nested_v = nested.value.cast(Node.Value).?;	294	const nested_v = nested.value.?.cast(Node.Value).?;
317	const leaf = tree.tokens[nested_v.start.?];	295	const leaf = tree.tokens[nested_v.base.start];
318	try testing.expectEqual(leaf.id, .Literal);	296	try testing.expectEqual(leaf.id, .literal);
319	try testing.expect(mem.eql(u8, "value2", tree.source[leaf.start..leaf.end]));	297	try testing.expectEqualStrings("value2", tree.source[leaf.start..leaf.end]);
320	}	298	}
321		299
322	{	300	{
323	const elem = value.values.items[1].cast(Node.Map).?;	301	const elem = value.values.items[1].cast(Node.Map).?;
324	const nested = elem.values.items[0];	302	const nested = elem.values.items[0];
325	const nested_key = tree.tokens[nested.key];	303	const nested_key = tree.tokens[nested.key];
326	try testing.expectEqual(nested_key.id, .Literal);	304	try testing.expectEqual(nested_key.id, .literal);
327	try testing.expect(mem.eql(u8, "key3", tree.source[nested_key.start..nested_key.end]));	305	try testing.expectEqualStrings("key3", tree.source[nested_key.start..nested_key.end]);
328		306
329	const nested_v = nested.value.cast(Node.Value).?;	307	const nested_v = nested.value.?.cast(Node.Value).?;
330	const leaf = tree.tokens[nested_v.start.?];	308	const leaf = tree.tokens[nested_v.base.start];
331	try testing.expectEqual(leaf.id, .Literal);	309	try testing.expectEqual(leaf.id, .literal);
332	try testing.expect(mem.eql(u8, "value3", tree.source[leaf.start..leaf.end]));	310	try testing.expectEqualStrings("value3", tree.source[leaf.start..leaf.end]);
333	}	311	}
334		312
335	{	313	{
336	const elem = value.values.items[2].cast(Node.Map).?;	314	const elem = value.values.items[2].cast(Node.Map).?;
337	const nested = elem.values.items[0];	315	const nested = elem.values.items[0];
338	const nested_key = tree.tokens[nested.key];	316	const nested_key = tree.tokens[nested.key];
339	try testing.expectEqual(nested_key.id, .Literal);	317	try testing.expectEqual(nested_key.id, .literal);
340	try testing.expect(mem.eql(u8, "key4", tree.source[nested_key.start..nested_key.end]));	318	try testing.expectEqualStrings("key4", tree.source[nested_key.start..nested_key.end]);
341		319
342	const nested_v = nested.value.cast(Node.Value).?;	320	const nested_v = nested.value.?.cast(Node.Value).?;
343	const leaf = tree.tokens[nested_v.start.?];	321	const leaf = tree.tokens[nested_v.base.start];
344	try testing.expectEqual(leaf.id, .Literal);	322	try testing.expectEqual(leaf.id, .literal);
345	try testing.expect(mem.eql(u8, "value4", tree.source[leaf.start..leaf.end]));	323	try testing.expectEqualStrings("value4", tree.source[leaf.start..leaf.end]);
346	}	324	}
347	}	325	}
348		326
...	@@ -360,15 +338,15 @@ test "list of lists" {	...	@@ -360,15 +338,15 @@ test "list of lists" {
360	try testing.expectEqual(tree.docs.items.len, 1);	338	try testing.expectEqual(tree.docs.items.len, 1);
361		339
362	const doc = tree.docs.items[0].cast(Node.Doc).?;	340	const doc = tree.docs.items[0].cast(Node.Doc).?;
363	try testing.expectEqual(doc.start.?, 0);	341	try testing.expectEqual(doc.base.start, 0);
364	try testing.expectEqual(doc.end.?, tree.tokens.len - 2);	342	try testing.expectEqual(doc.base.end, tree.tokens.len - 2);
365		343
366	try testing.expect(doc.value != null);	344	try testing.expect(doc.value != null);
367	try testing.expectEqual(doc.value.?.tag, .list);	345	try testing.expectEqual(doc.value.?.tag, .list);
368		346
369	const list = doc.value.?.cast(Node.List).?;	347	const list = doc.value.?.cast(Node.List).?;
370	try testing.expectEqual(list.start.?, 0);	348	try testing.expectEqual(list.base.start, 0);
371	try testing.expectEqual(list.end.?, tree.tokens.len - 2);	349	try testing.expectEqual(list.base.end, tree.tokens.len - 2);
372	try testing.expectEqual(list.values.items.len, 3);	350	try testing.expectEqual(list.values.items.len, 3);
373		351
374	{	352	{
...	@@ -379,22 +357,22 @@ test "list of lists" {	...	@@ -379,22 +357,22 @@ test "list of lists" {
379	{	357	{
380	try testing.expectEqual(nested.values.items[0].tag, .value);	358	try testing.expectEqual(nested.values.items[0].tag, .value);
381	const value = nested.values.items[0].cast(Node.Value).?;	359	const value = nested.values.items[0].cast(Node.Value).?;
382	const leaf = tree.tokens[value.start.?];	360	const leaf = tree.tokens[value.base.start];
383	try testing.expect(mem.eql(u8, "name", tree.source[leaf.start..leaf.end]));	361	try testing.expectEqualStrings("name", tree.source[leaf.start..leaf.end]);
384	}	362	}
385		363
386	{	364	{
387	try testing.expectEqual(nested.values.items[1].tag, .value);	365	try testing.expectEqual(nested.values.items[1].tag, .value);
388	const value = nested.values.items[1].cast(Node.Value).?;	366	const value = nested.values.items[1].cast(Node.Value).?;
389	const leaf = tree.tokens[value.start.?];	367	const leaf = tree.tokens[value.base.start];
390	try testing.expect(mem.eql(u8, "hr", tree.source[leaf.start..leaf.end]));	368	try testing.expectEqualStrings("hr", tree.source[leaf.start..leaf.end]);
391	}	369	}
392		370
393	{	371	{
394	try testing.expectEqual(nested.values.items[2].tag, .value);	372	try testing.expectEqual(nested.values.items[2].tag, .value);
395	const value = nested.values.items[2].cast(Node.Value).?;	373	const value = nested.values.items[2].cast(Node.Value).?;
396	const leaf = tree.tokens[value.start.?];	374	const leaf = tree.tokens[value.base.start];
397	try testing.expect(mem.eql(u8, "avg", tree.source[leaf.start..leaf.end]));	375	try testing.expectEqualStrings("avg", tree.source[leaf.start..leaf.end]);
398	}	376	}
399	}	377	}
400		378
...	@@ -406,23 +384,23 @@ test "list of lists" {	...	@@ -406,23 +384,23 @@ test "list of lists" {
406	{	384	{
407	try testing.expectEqual(nested.values.items[0].tag, .value);	385	try testing.expectEqual(nested.values.items[0].tag, .value);
408	const value = nested.values.items[0].cast(Node.Value).?;	386	const value = nested.values.items[0].cast(Node.Value).?;
409	const start = tree.tokens[value.start.?];	387	const start = tree.tokens[value.base.start];
410	const end = tree.tokens[value.end.?];	388	const end = tree.tokens[value.base.end];
411	try testing.expect(mem.eql(u8, "Mark McGwire", tree.source[start.start..end.end]));	389	try testing.expectEqualStrings("Mark McGwire", tree.source[start.start..end.end]);
412	}	390	}
413		391
414	{	392	{
415	try testing.expectEqual(nested.values.items[1].tag, .value);	393	try testing.expectEqual(nested.values.items[1].tag, .value);
416	const value = nested.values.items[1].cast(Node.Value).?;	394	const value = nested.values.items[1].cast(Node.Value).?;
417	const leaf = tree.tokens[value.start.?];	395	const leaf = tree.tokens[value.base.start];
418	try testing.expect(mem.eql(u8, "65", tree.source[leaf.start..leaf.end]));	396	try testing.expectEqualStrings("65", tree.source[leaf.start..leaf.end]);
419	}	397	}
420		398
421	{	399	{
422	try testing.expectEqual(nested.values.items[2].tag, .value);	400	try testing.expectEqual(nested.values.items[2].tag, .value);
423	const value = nested.values.items[2].cast(Node.Value).?;	401	const value = nested.values.items[2].cast(Node.Value).?;
424	const leaf = tree.tokens[value.start.?];	402	const leaf = tree.tokens[value.base.start];
425	try testing.expect(mem.eql(u8, "0.278", tree.source[leaf.start..leaf.end]));	403	try testing.expectEqualStrings("0.278", tree.source[leaf.start..leaf.end]);
426	}	404	}
427	}	405	}
428		406
...	@@ -434,23 +412,23 @@ test "list of lists" {	...	@@ -434,23 +412,23 @@ test "list of lists" {
434	{	412	{
435	try testing.expectEqual(nested.values.items[0].tag, .value);	413	try testing.expectEqual(nested.values.items[0].tag, .value);
436	const value = nested.values.items[0].cast(Node.Value).?;	414	const value = nested.values.items[0].cast(Node.Value).?;
437	const start = tree.tokens[value.start.?];	415	const start = tree.tokens[value.base.start];
438	const end = tree.tokens[value.end.?];	416	const end = tree.tokens[value.base.end];
439	try testing.expect(mem.eql(u8, "Sammy Sosa", tree.source[start.start..end.end]));	417	try testing.expectEqualStrings("Sammy Sosa", tree.source[start.start..end.end]);
440	}	418	}
441		419
442	{	420	{
443	try testing.expectEqual(nested.values.items[1].tag, .value);	421	try testing.expectEqual(nested.values.items[1].tag, .value);
444	const value = nested.values.items[1].cast(Node.Value).?;	422	const value = nested.values.items[1].cast(Node.Value).?;
445	const leaf = tree.tokens[value.start.?];	423	const leaf = tree.tokens[value.base.start];
446	try testing.expect(mem.eql(u8, "63", tree.source[leaf.start..leaf.end]));	424	try testing.expectEqualStrings("63", tree.source[leaf.start..leaf.end]);
447	}	425	}
448		426
449	{	427	{
450	try testing.expectEqual(nested.values.items[2].tag, .value);	428	try testing.expectEqual(nested.values.items[2].tag, .value);
451	const value = nested.values.items[2].cast(Node.Value).?;	429	const value = nested.values.items[2].cast(Node.Value).?;
452	const leaf = tree.tokens[value.start.?];	430	const leaf = tree.tokens[value.base.start];
453	try testing.expect(mem.eql(u8, "0.288", tree.source[leaf.start..leaf.end]));	431	try testing.expectEqualStrings("0.288", tree.source[leaf.start..leaf.end]);
454	}	432	}
455	}	433	}
456	}	434	}
...	@@ -467,36 +445,36 @@ test "inline list" {	...	@@ -467,36 +445,36 @@ test "inline list" {
467	try testing.expectEqual(tree.docs.items.len, 1);	445	try testing.expectEqual(tree.docs.items.len, 1);
468		446
469	const doc = tree.docs.items[0].cast(Node.Doc).?;	447	const doc = tree.docs.items[0].cast(Node.Doc).?;
470	try testing.expectEqual(doc.start.?, 0);	448	try testing.expectEqual(doc.base.start, 0);
471	try testing.expectEqual(doc.end.?, tree.tokens.len - 2);	449	try testing.expectEqual(doc.base.end, tree.tokens.len - 2);
472		450
473	try testing.expect(doc.value != null);	451	try testing.expect(doc.value != null);
474	try testing.expectEqual(doc.value.?.tag, .list);	452	try testing.expectEqual(doc.value.?.tag, .list);
475		453
476	const list = doc.value.?.cast(Node.List).?;	454	const list = doc.value.?.cast(Node.List).?;
477	try testing.expectEqual(list.start.?, 0);	455	try testing.expectEqual(list.base.start, 0);
478	try testing.expectEqual(list.end.?, tree.tokens.len - 2);	456	try testing.expectEqual(list.base.end, tree.tokens.len - 2);
479	try testing.expectEqual(list.values.items.len, 3);	457	try testing.expectEqual(list.values.items.len, 3);
480		458
481	{	459	{
482	try testing.expectEqual(list.values.items[0].tag, .value);	460	try testing.expectEqual(list.values.items[0].tag, .value);
483	const value = list.values.items[0].cast(Node.Value).?;	461	const value = list.values.items[0].cast(Node.Value).?;
484	const leaf = tree.tokens[value.start.?];	462	const leaf = tree.tokens[value.base.start];
485	try testing.expect(mem.eql(u8, "name", tree.source[leaf.start..leaf.end]));	463	try testing.expectEqualStrings("name", tree.source[leaf.start..leaf.end]);
486	}	464	}
487		465
488	{	466	{
489	try testing.expectEqual(list.values.items[1].tag, .value);	467	try testing.expectEqual(list.values.items[1].tag, .value);
490	const value = list.values.items[1].cast(Node.Value).?;	468	const value = list.values.items[1].cast(Node.Value).?;
491	const leaf = tree.tokens[value.start.?];	469	const leaf = tree.tokens[value.base.start];
492	try testing.expect(mem.eql(u8, "hr", tree.source[leaf.start..leaf.end]));	470	try testing.expectEqualStrings("hr", tree.source[leaf.start..leaf.end]);
493	}	471	}
494		472
495	{	473	{
496	try testing.expectEqual(list.values.items[2].tag, .value);	474	try testing.expectEqual(list.values.items[2].tag, .value);
497	const value = list.values.items[2].cast(Node.Value).?;	475	const value = list.values.items[2].cast(Node.Value).?;
498	const leaf = tree.tokens[value.start.?];	476	const leaf = tree.tokens[value.base.start];
499	try testing.expect(mem.eql(u8, "avg", tree.source[leaf.start..leaf.end]));	477	try testing.expectEqualStrings("avg", tree.source[leaf.start..leaf.end]);
500	}	478	}
501	}	479	}
502		480
...	@@ -514,45 +492,273 @@ test "inline list as mapping value" {	...	@@ -514,45 +492,273 @@ test "inline list as mapping value" {
514	try testing.expectEqual(tree.docs.items.len, 1);	492	try testing.expectEqual(tree.docs.items.len, 1);
515		493
516	const doc = tree.docs.items[0].cast(Node.Doc).?;	494	const doc = tree.docs.items[0].cast(Node.Doc).?;
517	try testing.expectEqual(doc.start.?, 0);	495	try testing.expectEqual(doc.base.start, 0);
518	try testing.expectEqual(doc.end.?, tree.tokens.len - 2);	496	try testing.expectEqual(doc.base.end, tree.tokens.len - 2);
519		497
520	try testing.expect(doc.value != null);	498	try testing.expect(doc.value != null);
521	try testing.expectEqual(doc.value.?.tag, .map);	499	try testing.expectEqual(doc.value.?.tag, .map);
522		500
523	const map = doc.value.?.cast(Node.Map).?;	501	const map = doc.value.?.cast(Node.Map).?;
524	try testing.expectEqual(map.start.?, 0);	502	try testing.expectEqual(map.base.start, 0);
525	try testing.expectEqual(map.end.?, tree.tokens.len - 2);	503	try testing.expectEqual(map.base.end, tree.tokens.len - 2);
526	try testing.expectEqual(map.values.items.len, 1);	504	try testing.expectEqual(map.values.items.len, 1);
527		505
528	const entry = map.values.items[0];	506	const entry = map.values.items[0];
529	const key = tree.tokens[entry.key];	507	const key = tree.tokens[entry.key];
530	try testing.expectEqual(key.id, .Literal);	508	try testing.expectEqual(key.id, .literal);
531	try testing.expect(mem.eql(u8, "key", tree.source[key.start..key.end]));	509	try testing.expectEqualStrings("key", tree.source[key.start..key.end]);
532		510
533	const list = entry.value.cast(Node.List).?;	511	const list = entry.value.?.cast(Node.List).?;
534	try testing.expectEqual(list.start.?, 4);	512	try testing.expectEqual(list.base.start, 4);
535	try testing.expectEqual(list.end.?, tree.tokens.len - 2);	513	try testing.expectEqual(list.base.end, tree.tokens.len - 2);
536	try testing.expectEqual(list.values.items.len, 3);	514	try testing.expectEqual(list.values.items.len, 3);
537		515
538	{	516	{
539	try testing.expectEqual(list.values.items[0].tag, .value);	517	try testing.expectEqual(list.values.items[0].tag, .value);
540	const value = list.values.items[0].cast(Node.Value).?;	518	const value = list.values.items[0].cast(Node.Value).?;
541	const leaf = tree.tokens[value.start.?];	519	const leaf = tree.tokens[value.base.start];
542	try testing.expect(mem.eql(u8, "name", tree.source[leaf.start..leaf.end]));	520	try testing.expectEqualStrings("name", tree.source[leaf.start..leaf.end]);
543	}	521	}
544		522
545	{	523	{
546	try testing.expectEqual(list.values.items[1].tag, .value);	524	try testing.expectEqual(list.values.items[1].tag, .value);
547	const value = list.values.items[1].cast(Node.Value).?;	525	const value = list.values.items[1].cast(Node.Value).?;
548	const leaf = tree.tokens[value.start.?];	526	const leaf = tree.tokens[value.base.start];
549	try testing.expect(mem.eql(u8, "hr", tree.source[leaf.start..leaf.end]));	527	try testing.expectEqualStrings("hr", tree.source[leaf.start..leaf.end]);
550	}	528	}
551		529
552	{	530	{
553	try testing.expectEqual(list.values.items[2].tag, .value);	531	try testing.expectEqual(list.values.items[2].tag, .value);
554	const value = list.values.items[2].cast(Node.Value).?;	532	const value = list.values.items[2].cast(Node.Value).?;
555	const leaf = tree.tokens[value.start.?];	533	const leaf = tree.tokens[value.base.start];
556	try testing.expect(mem.eql(u8, "avg", tree.source[leaf.start..leaf.end]));	534	try testing.expectEqualStrings("avg", tree.source[leaf.start..leaf.end]);
557	}	535	}
558	}	536	}
		537
		538	fn parseSuccess(comptime source: []const u8) !void {
		539	var tree = Tree.init(testing.allocator);
		540	defer tree.deinit();
		541	try tree.parse(source);
		542	}
		543
		544	fn parseError(comptime source: []const u8, err: parse.ParseError) !void {
		545	var tree = Tree.init(testing.allocator);
		546	defer tree.deinit();
		547	try testing.expectError(err, tree.parse(source));
		548	}
		549
		550	test "empty doc with spaces and comments" {
		551	try parseSuccess(
		552	\\
		553	\\
		554	\\ # this is a comment in a weird place
		555	\\# and this one is too
		556	);
		557	}
		558
		559	test "comment between --- and ! in document start" {
		560	try parseError(
		561	\\--- # what is it?
		562	\\!
		563	, error.UnexpectedToken);
		564	}
		565
		566	test "correct doc start with tag" {
		567	try parseSuccess(
		568	\\--- !some-tag
		569	\\
		570	);
		571	}
		572
		573	test "doc close without explicit doc open" {
		574	try parseError(
		575	\\
		576	\\
		577	\\# something cool
		578	\\...
		579	, error.UnexpectedToken);
		580	}
		581
		582	test "doc open and close are ok" {
		583	try parseSuccess(
		584	\\---
		585	\\# first doc
		586	\\
		587	\\
		588	\\---
		589	\\# second doc
		590	\\
		591	\\
		592	\\...
		593	);
		594	}
		595
		596	test "doc with a single string is ok" {
		597	try parseSuccess(
		598	\\a string of some sort
		599	\\
		600	);
		601	}
		602
		603	test "explicit doc with a single string is ok" {
		604	try parseSuccess(
		605	\\--- !anchor
		606	\\# nothing to see here except one string
		607	\\ # not a lot to go on with
		608	\\a single string
		609	\\...
		610	);
		611	}
		612
		613	test "doc with two string is bad" {
		614	try parseError(
		615	\\first
		616	\\second
		617	\\# this should fail already
		618	, error.UnexpectedToken);
		619	}
		620
		621	test "single quote string can have new lines" {
		622	try parseSuccess(
		623	\\'what is this
		624	\\ thing?'
		625	);
		626	}
		627
		628	test "single quote string on one line is fine" {
		629	try parseSuccess(
		630	\\'here''s an apostrophe'
		631	);
		632	}
		633
		634	test "double quote string can have new lines" {
		635	try parseSuccess(
		636	\\"what is this
		637	\\ thing?"
		638	);
		639	}
		640
		641	test "double quote string on one line is fine" {
		642	try parseSuccess(
		643	\\"a newline\nand a\ttab"
		644	);
		645	}
		646
		647	test "map with key and value literals" {
		648	try parseSuccess(
		649	\\key1: val1
		650	\\key2 : val2
		651	);
		652	}
		653
		654	test "map of maps" {
		655	try parseSuccess(
		656	\\
		657	\\# the first key
		658	\\key1:
		659	\\ # the first subkey
		660	\\ key1_1: 0
		661	\\ key1_2: 1
		662	\\# the second key
		663	\\key2:
		664	\\ key2_1: -1
		665	\\ key2_2: -2
		666	\\# the end of map
		667	);
		668	}
		669
		670	test "map value indicator needs to be on the same line" {
		671	try parseError(
		672	\\a
		673	\\ : b
		674	, error.UnexpectedToken);
		675	}
		676
		677	test "value needs to be indented" {
		678	try parseError(
		679	\\a:
		680	\\b
		681	, error.MalformedYaml);
		682	}
		683
		684	test "comment between a key and a value is fine" {
		685	try parseSuccess(
		686	\\a:
		687	\\ # this is a value
		688	\\ b
		689	);
		690	}
		691
		692	test "simple list" {
		693	try parseSuccess(
		694	\\# first el
		695	\\- a
		696	\\# second el
		697	\\- b
		698	\\# third el
		699	\\- c
		700	);
		701	}
		702
		703	test "list indentation matters" {
		704	try parseSuccess(
		705	\\ - a
		706	\\- b
		707	);
		708
		709	try parseSuccess(
		710	\\- a
		711	\\ - b
		712	);
		713	}
		714
		715	test "unindented list is fine too" {
		716	try parseSuccess(
		717	\\a:
		718	\\- 0
		719	\\- 1
		720	);
		721	}
		722
		723	test "empty values in a map" {
		724	try parseSuccess(
		725	\\a:
		726	\\b:
		727	\\- 0
		728	);
		729	}
		730
		731	test "weirdly nested map of maps of lists" {
		732	try parseSuccess(
		733	\\a:
		734	\\ b:
		735	\\ - 0
		736	\\ - 1
		737	);
		738	}
		739
		740	test "square brackets denote a list" {
		741	try parseSuccess(
		742	\\[ a,
		743	\\ b, c ]
		744	);
		745	}
		746
		747	test "empty list" {
		748	try parseSuccess(
		749	\\[ ]
		750	);
		751	}
		752
		753	test "comment within a bracketed list is an error" {
		754	try parseError(
		755	\\[ # something
		756	\\]
		757	, error.MalformedYaml);
		758	}
		759
		760	test "mixed ints with floats in a list" {
		761	try parseSuccess(
		762	\\[0, 1.0]
		763	);
		764	}

src/link/tapi/yaml.zig+176-396

...	@@ -2,8 +2,7 @@ const std = @import("std");	...	@@ -2,8 +2,7 @@ const std = @import("std");
2	const assert = std.debug.assert;	2	const assert = std.debug.assert;
3	const math = std.math;	3	const math = std.math;
4	const mem = std.mem;	4	const mem = std.mem;
5	const testing = std.testing;	5	const log = std.log.scoped(.yaml);
6	const log = std.log.scoped(.tapi);
7		6
8	const Allocator = mem.Allocator;	7	const Allocator = mem.Allocator;
9	const ArenaAllocator = std.heap.ArenaAllocator;	8	const ArenaAllocator = std.heap.ArenaAllocator;
...	@@ -17,22 +16,15 @@ const ParseError = parse.ParseError;	...	@@ -17,22 +16,15 @@ const ParseError = parse.ParseError;
17		16
18	pub const YamlError = error{	17	pub const YamlError = error{
19	UnexpectedNodeType,	18	UnexpectedNodeType,
		19	DuplicateMapKey,
20	OutOfMemory,	20	OutOfMemory,
		21	CannotEncodeValue,
21	} \|\| ParseError \|\| std.fmt.ParseIntError;	22	} \|\| ParseError \|\| std.fmt.ParseIntError;
22		23
23	pub const ValueType = enum {
24	empty,
25	int,
26	float,
27	string,
28	list,
29	map,
30	};
31
32	pub const List = []Value;	24	pub const List = []Value;
33	pub const Map = std.StringArrayHashMap(Value);	25	pub const Map = std.StringHashMap(Value);
34		26
35	pub const Value = union(ValueType) {	27	pub const Value = union(enum) {
36	empty,	28	empty,
37	int: i64,	29	int: i64,
38	float: f64,	30	float: f64,
...	@@ -70,9 +62,7 @@ pub const Value = union(ValueType) {	...	@@ -70,9 +62,7 @@ pub const Value = union(ValueType) {
70	should_inline_first_key: bool = false,	62	should_inline_first_key: bool = false,
71	};	63	};
72		64
73	pub const StringifyError = std.os.WriteError;	65	pub fn stringify(self: Value, writer: anytype, args: StringifyArgs) anyerror!void {
74
75	pub fn stringify(self: Value, writer: anytype, args: StringifyArgs) StringifyError!void {
76	switch (self) {	66	switch (self) {
77	.empty => return,	67	.empty => return,
78	.int => \|int\| return writer.print("{}", .{int}),	68	.int => \|int\| return writer.print("{}", .{int}),
...	@@ -83,7 +73,7 @@ pub const Value = union(ValueType) {	...	@@ -83,7 +73,7 @@ pub const Value = union(ValueType) {
83	if (len == 0) return;	73	if (len == 0) return;
84		74
85	const first = list[0];	75	const first = list[0];
86	if (first.is_compound()) {	76	if (first.isCompound()) {
87	for (list, 0..) \|elem, i\| {	77	for (list, 0..) \|elem, i\| {
88	try writer.writeByteNTimes(' ', args.indentation);	78	try writer.writeByteNTimes(' ', args.indentation);
89	try writer.writeAll("- ");	79	try writer.writeAll("- ");
...	@@ -108,20 +98,23 @@ pub const Value = union(ValueType) {	...	@@ -108,20 +98,23 @@ pub const Value = union(ValueType) {
108	try writer.writeAll(" ]");	98	try writer.writeAll(" ]");
109	},	99	},
110	.map => \|map\| {	100	.map => \|map\| {
111	const keys = map.keys();	101	const len = map.count();
112	const len = keys.len;
113	if (len == 0) return;	102	if (len == 0) return;
114		103
115	for (keys, 0..) \|key, i\| {	104	var i: usize = 0;
		105	var it = map.iterator();
		106	while (it.next()) \|entry\| {
		107	const key = entry.key_ptr.*;
		108	const value = entry.value_ptr.*;
		109
116	if (!args.should_inline_first_key or i != 0) {	110	if (!args.should_inline_first_key or i != 0) {
117	try writer.writeByteNTimes(' ', args.indentation);	111	try writer.writeByteNTimes(' ', args.indentation);
118	}	112	}
119	try writer.print("{s}: ", .{key});	113	try writer.print("{s}: ", .{key});
120		114
121	const value = map.get(key) orelse unreachable;
122	const should_inline = blk: {	115	const should_inline = blk: {
123	if (!value.is_compound()) break :blk true;	116	if (!value.isCompound()) break :blk true;
124	if (value == .list and value.list.len > 0 and !value.list[0].is_compound()) break :blk true;	117	if (value == .list and value.list.len > 0 and !value.list[0].isCompound()) break :blk true;
125	break :blk false;	118	break :blk false;
126	};	119	};
127		120
...	@@ -137,35 +130,44 @@ pub const Value = union(ValueType) {	...	@@ -137,35 +130,44 @@ pub const Value = union(ValueType) {
137	if (i < len - 1) {	130	if (i < len - 1) {
138	try writer.writeByte('\n');	131	try writer.writeByte('\n');
139	}	132	}
		133
		134	i += 1;
140	}	135	}
141	},	136	},
142	}	137	}
143	}	138	}
144		139
145	fn is_compound(self: Value) bool {	140	fn isCompound(self: Value) bool {
146	return switch (self) {	141	return switch (self) {
147	.list, .map => true,	142	.list, .map => true,
148	else => false,	143	else => false,
149	};	144	};
150	}	145	}
151		146
152	fn fromNode(arena: Allocator, tree: const Tree, node: const Node, type_hint: ?ValueType) YamlError!Value {	147	fn fromNode(arena: Allocator, tree: const Tree, node: const Node) YamlError!Value {
153	if (node.cast(Node.Doc)) \|doc\| {	148	if (node.cast(Node.Doc)) \|doc\| {
154	const inner = doc.value orelse {	149	const inner = doc.value orelse {
155	// empty doc	150	// empty doc
156	return Value{ .empty = {} };	151	return Value{ .empty = {} };
157	};	152	};
158	return Value.fromNode(arena, tree, inner, null);	153	return Value.fromNode(arena, tree, inner);
159	} else if (node.cast(Node.Map)) \|map\| {	154	} else if (node.cast(Node.Map)) \|map\| {
160	var out_map = std.StringArrayHashMap(Value).init(arena);	155	// TODO use ContextAdapted HashMap and do not duplicate keys, intern
161	try out_map.ensureUnusedCapacity(map.values.items.len);	156	// in a contiguous string buffer.
		157	var out_map = std.StringHashMap(Value).init(arena);
		158	try out_map.ensureUnusedCapacity(math.cast(u32, map.values.items.len) orelse return error.Overflow);
162		159
163	for (map.values.items) \|entry\| {	160	for (map.values.items) \|entry\| {
164	const key_tok = tree.tokens[entry.key];	161	const key = try arena.dupe(u8, tree.getRaw(entry.key, entry.key));
165	const key = try arena.dupe(u8, tree.source[key_tok.start..key_tok.end]);	162	const gop = out_map.getOrPutAssumeCapacity(key);
166	const value = try Value.fromNode(arena, tree, entry.value, null);	163	if (gop.found_existing) {
167		164	return error.DuplicateMapKey;
168	out_map.putAssumeCapacityNoClobber(key, value);	165	}
		166	const value = if (entry.value) \|value\|
		167	try Value.fromNode(arena, tree, value)
		168	else
		169	.empty;
		170	gop.value_ptr.* = value;
169	}	171	}
170		172
171	return Value{ .map = out_map };	173	return Value{ .map = out_map };
...	@@ -173,56 +175,124 @@ pub const Value = union(ValueType) {	...	@@ -173,56 +175,124 @@ pub const Value = union(ValueType) {
173	var out_list = std.ArrayList(Value).init(arena);	175	var out_list = std.ArrayList(Value).init(arena);
174	try out_list.ensureUnusedCapacity(list.values.items.len);	176	try out_list.ensureUnusedCapacity(list.values.items.len);
175		177
176	if (list.values.items.len > 0) {	178	for (list.values.items) \|elem\| {
177	const hint = if (list.values.items[0].cast(Node.Value)) \|value\| hint: {	179	const value = try Value.fromNode(arena, tree, elem);
178	const start = tree.tokens[value.start.?];	180	out_list.appendAssumeCapacity(value);
179	const end = tree.tokens[value.end.?];
180	const raw = tree.source[start.start..end.end];
181	_ = std.fmt.parseInt(i64, raw, 10) catch {
182	_ = std.fmt.parseFloat(f64, raw) catch {
183	break :hint ValueType.string;
184	};
185	break :hint ValueType.float;
186	};
187	break :hint ValueType.int;
188	} else null;
189
190	for (list.values.items) \|elem\| {
191	const value = try Value.fromNode(arena, tree, elem, hint);
192	out_list.appendAssumeCapacity(value);
193	}
194	}	181	}
195		182
196	return Value{ .list = try out_list.toOwnedSlice() };	183	return Value{ .list = try out_list.toOwnedSlice() };
197	} else if (node.cast(Node.Value)) \|value\| {	184	} else if (node.cast(Node.Value)) \|value\| {
198	const start = tree.tokens[value.start.?];	185	const raw = tree.getRaw(node.start, node.end);
199	const end = tree.tokens[value.end.?];
200	const raw = tree.source[start.start..end.end];
201
202	if (type_hint) \|hint\| {
203	return switch (hint) {
204	.int => Value{ .int = try std.fmt.parseInt(i64, raw, 10) },
205	.float => Value{ .float = try std.fmt.parseFloat(f64, raw) },
206	.string => Value{ .string = try arena.dupe(u8, raw) },
207	else => unreachable,
208	};
209	}
210		186
211	try_int: {	187	try_int: {
212	// TODO infer base for int	188	// TODO infer base for int
213	const int = std.fmt.parseInt(i64, raw, 10) catch break :try_int;	189	const int = std.fmt.parseInt(i64, raw, 10) catch break :try_int;
214	return Value{ .int = int };	190	return Value{ .int = int };
215	}	191	}
		192
216	try_float: {	193	try_float: {
217	const float = std.fmt.parseFloat(f64, raw) catch break :try_float;	194	const float = std.fmt.parseFloat(f64, raw) catch break :try_float;
218	return Value{ .float = float };	195	return Value{ .float = float };
219	}	196	}
220	return Value{ .string = try arena.dupe(u8, raw) };	197
		198	return Value{ .string = try arena.dupe(u8, value.string_value.items) };
221	} else {	199	} else {
222	log.err("Unexpected node type: {}", .{node.tag});	200	log.err("Unexpected node type: {}", .{node.tag});
223	return error.UnexpectedNodeType;	201	return error.UnexpectedNodeType;
224	}	202	}
225	}	203	}
		204
		205	fn encode(arena: Allocator, input: anytype) YamlError!?Value {
		206	switch (@typeInfo(@TypeOf(input))) {
		207	.ComptimeInt,
		208	.Int,
		209	=> return Value{ .int = math.cast(i64, input) orelse return error.Overflow },
		210
		211	.Float => return Value{ .float = math.lossyCast(f64, input) },
		212
		213	.Struct => \|info\| if (info.is_tuple) {
		214	var list = std.ArrayList(Value).init(arena);
		215	errdefer list.deinit();
		216	try list.ensureTotalCapacityPrecise(info.fields.len);
		217
		218	inline for (info.fields) \|field\| {
		219	if (try encode(arena, @field(input, field.name))) \|value\| {
		220	list.appendAssumeCapacity(value);
		221	}
		222	}
		223
		224	return Value{ .list = try list.toOwnedSlice() };
		225	} else {
		226	var map = Map.init(arena);
		227	errdefer map.deinit();
		228	try map.ensureTotalCapacity(info.fields.len);
		229
		230	inline for (info.fields) \|field\| {
		231	if (try encode(arena, @field(input, field.name))) \|value\| {
		232	const key = try arena.dupe(u8, field.name);
		233	map.putAssumeCapacityNoClobber(key, value);
		234	}
		235	}
		236
		237	return Value{ .map = map };
		238	},
		239
		240	.Union => \|info\| if (info.tag_type) \|tag_type\| {
		241	inline for (info.fields) \|field\| {
		242	if (@field(tag_type, field.name) == input) {
		243	return try encode(arena, @field(input, field.name));
		244	}
		245	} else unreachable;
		246	} else return error.UntaggedUnion,
		247
		248	.Array => return encode(arena, &input),
		249
		250	.Pointer => \|info\| switch (info.size) {
		251	.One => switch (@typeInfo(info.child)) {
		252	.Array => \|child_info\| {
		253	const Slice = []const child_info.child;
		254	return encode(arena, @as(Slice, input));
		255	},
		256	else => {
		257	@compileError("Unhandled type: {s}" ++ @typeName(info.child));
		258	},
		259	},
		260	.Slice => {
		261	if (info.child == u8) {
		262	return Value{ .string = try arena.dupe(u8, input) };
		263	}
		264
		265	var list = std.ArrayList(Value).init(arena);
		266	errdefer list.deinit();
		267	try list.ensureTotalCapacityPrecise(input.len);
		268
		269	for (input) \|elem\| {
		270	if (try encode(arena, elem)) \|value\| {
		271	list.appendAssumeCapacity(value);
		272	} else {
		273	log.err("Could not encode value in a list: {any}", .{elem});
		274	return error.CannotEncodeValue;
		275	}
		276	}
		277
		278	return Value{ .list = try list.toOwnedSlice() };
		279	},
		280	else => {
		281	@compileError("Unhandled type: {s}" ++ @typeName(@TypeOf(input)));
		282	},
		283	},
		284
		285	// TODO we should probably have an option to encode `null` and also
		286	// allow for some default value too.
		287	.Optional => return if (input) \|val\| encode(arena, val) else null,
		288
		289	.Null => return null,
		290
		291	else => {
		292	@compileError("Unhandled type: {s}" ++ @typeName(@TypeOf(input)));
		293	},
		294	}
		295	}
226	};	296	};
227		297
228	pub const Yaml = struct {	298	pub const Yaml = struct {
...	@@ -234,30 +304,18 @@ pub const Yaml = struct {	...	@@ -234,30 +304,18 @@ pub const Yaml = struct {
234	self.arena.deinit();	304	self.arena.deinit();
235	}	305	}
236		306
237	pub fn stringify(self: Yaml, writer: anytype) !void {
238	for (self.docs.items) \|doc\| {
239	// if (doc.directive) \|directive\| {
240	// try writer.print("--- !{s}\n", .{directive});
241	// }
242	try doc.stringify(writer, .{});
243	// if (doc.directive != null) {
244	// try writer.writeAll("...\n");
245	// }
246	}
247	}
248
249	pub fn load(allocator: Allocator, source: []const u8) !Yaml {	307	pub fn load(allocator: Allocator, source: []const u8) !Yaml {
250	var arena = ArenaAllocator.init(allocator);	308	var arena = ArenaAllocator.init(allocator);
251	const arena_allocator = arena.allocator();	309	errdefer arena.deinit();
252		310
253	var tree = Tree.init(arena_allocator);	311	var tree = Tree.init(arena.allocator());
254	try tree.parse(source);	312	try tree.parse(source);
255		313
256	var docs = std.ArrayList(Value).init(arena_allocator);	314	var docs = std.ArrayList(Value).init(arena.allocator());
257	try docs.ensureUnusedCapacity(tree.docs.items.len);	315	try docs.ensureTotalCapacityPrecise(tree.docs.items.len);
258		316
259	for (tree.docs.items) \|node\| {	317	for (tree.docs.items) \|node\| {
260	const value = try Value.fromNode(arena_allocator, &tree, node, null);	318	const value = try Value.fromNode(arena.allocator(), &tree, node);
261	docs.appendAssumeCapacity(value);	319	docs.appendAssumeCapacity(value);
262	}	320	}
263		321
...	@@ -316,17 +374,19 @@ pub const Yaml = struct {	...	@@ -316,17 +374,19 @@ pub const Yaml = struct {
316		374
317	fn parseValue(self: *Yaml, comptime T: type, value: Value) Error!T {	375	fn parseValue(self: *Yaml, comptime T: type, value: Value) Error!T {
318	return switch (@typeInfo(T)) {	376	return switch (@typeInfo(T)) {
319	.Int => math.cast(T, try value.asInt()) orelse error.Overflow,	377	.Int => math.cast(T, try value.asInt()) orelse return error.Overflow,
320	.Float => math.lossyCast(T, try value.asFloat()),	378	.Float => if (value.asFloat()) \|float\| {
		379	return math.lossyCast(T, float);
		380	} else \|_\| {
		381	return math.lossyCast(T, try value.asInt());
		382	},
321	.Struct => self.parseStruct(T, try value.asMap()),	383	.Struct => self.parseStruct(T, try value.asMap()),
322	.Union => self.parseUnion(T, value),	384	.Union => self.parseUnion(T, value),
323	.Array => self.parseArray(T, try value.asList()),	385	.Array => self.parseArray(T, try value.asList()),
324	.Pointer => {	386	.Pointer => if (value.asList()) \|list\| {
325	if (value.asList()) \|list\| {	387	return self.parsePointer(T, .{ .list = list });
326	return self.parsePointer(T, .{ .list = list });	388	} else \|_\| {
327	} else \|_\| {	389	return self.parsePointer(T, .{ .string = try value.asString() });
328	return self.parsePointer(T, .{ .string = try value.asString() });
329	}
330	},	390	},
331	.Void => error.TypeMismatch,	391	.Void => error.TypeMismatch,
332	.Optional => unreachable,	392	.Optional => unreachable,
...	@@ -372,7 +432,7 @@ pub const Yaml = struct {	...	@@ -372,7 +432,7 @@ pub const Yaml = struct {
372	}	432	}
373		433
374	const unwrapped = value orelse {	434	const unwrapped = value orelse {
375	log.debug("missing struct field: {s}: {s}", .{ field.name, @typeName(field.type) });	435	log.err("missing struct field: {s}: {s}", .{ field.name, @typeName(field.type) });
376	return error.StructFieldMissing;	436	return error.StructFieldMissing;
377	};	437	};
378	@field(parsed, field.name) = try self.parseValue(field.type, unwrapped);	438	@field(parsed, field.name) = try self.parseValue(field.type, unwrapped);
...	@@ -387,8 +447,7 @@ pub const Yaml = struct {	...	@@ -387,8 +447,7 @@ pub const Yaml = struct {
387		447
388	switch (ptr_info.size) {	448	switch (ptr_info.size) {
389	.Slice => {	449	.Slice => {
390	const child_info = @typeInfo(ptr_info.child);	450	if (ptr_info.child == u8) {
391	if (child_info == .Int and child_info.Int.bits == 8) {
392	return value.asString();	451	return value.asString();
393	}	452	}
394		453
...	@@ -413,315 +472,36 @@ pub const Yaml = struct {	...	@@ -413,315 +472,36 @@ pub const Yaml = struct {
413		472
414	return parsed;	473	return parsed;
415	}	474	}
416	};
417
418	test {
419	testing.refAllDecls(@This());
420	}
421
422	test "simple list" {
423	const source =
424	\\- a
425	\\- b
426	\\- c
427	;
428
429	var yaml = try Yaml.load(testing.allocator, source);
430	defer yaml.deinit();
431		475
432	try testing.expectEqual(yaml.docs.items.len, 1);	476	pub fn stringify(self: Yaml, writer: anytype) !void {
433		477	for (self.docs.items, 0..) \|doc, i\| {
434	const list = yaml.docs.items[0].list;	478	try writer.writeAll("---");
435	try testing.expectEqual(list.len, 3);	479	if (self.tree.?.getDirective(i)) \|directive\| {
436		480	try writer.print(" !{s}", .{directive});
437	try testing.expect(mem.eql(u8, list[0].string, "a"));	481	}
438	try testing.expect(mem.eql(u8, list[1].string, "b"));	482	try writer.writeByte('\n');
439	try testing.expect(mem.eql(u8, list[2].string, "c"));	483	try doc.stringify(writer, .{});
440	}	484	try writer.writeByte('\n');
441		485	}
442	test "simple list typed as array of strings" {	486	try writer.writeAll("...\n");
443	const source =
444	\\- a
445	\\- b
446	\\- c
447	;
448
449	var yaml = try Yaml.load(testing.allocator, source);
450	defer yaml.deinit();
451
452	try testing.expectEqual(yaml.docs.items.len, 1);
453
454	const arr = try yaml.parse([3][]const u8);
455	try testing.expectEqual(arr.len, 3);
456	try testing.expect(mem.eql(u8, arr[0], "a"));
457	try testing.expect(mem.eql(u8, arr[1], "b"));
458	try testing.expect(mem.eql(u8, arr[2], "c"));
459	}
460
461	test "simple list typed as array of ints" {
462	const source =
463	\\- 0
464	\\- 1
465	\\- 2
466	;
467
468	var yaml = try Yaml.load(testing.allocator, source);
469	defer yaml.deinit();
470
471	try testing.expectEqual(yaml.docs.items.len, 1);
472
473	const arr = try yaml.parse([3]u8);
474	try testing.expectEqual(arr.len, 3);
475	try testing.expectEqual(arr[0], 0);
476	try testing.expectEqual(arr[1], 1);
477	try testing.expectEqual(arr[2], 2);
478	}
479
480	test "list of mixed sign integer" {
481	const source =
482	\\- 0
483	\\- -1
484	\\- 2
485	;
486
487	var yaml = try Yaml.load(testing.allocator, source);
488	defer yaml.deinit();
489
490	try testing.expectEqual(yaml.docs.items.len, 1);
491
492	const arr = try yaml.parse([3]i8);
493	try testing.expectEqual(arr.len, 3);
494	try testing.expectEqual(arr[0], 0);
495	try testing.expectEqual(arr[1], -1);
496	try testing.expectEqual(arr[2], 2);
497	}
498
499	test "simple map untyped" {
500	const source =
501	\\a: 0
502	;
503
504	var yaml = try Yaml.load(testing.allocator, source);
505	defer yaml.deinit();
506
507	try testing.expectEqual(yaml.docs.items.len, 1);
508
509	const map = yaml.docs.items[0].map;
510	try testing.expect(map.contains("a"));
511	try testing.expectEqual(map.get("a").?.int, 0);
512	}
513
514	test "simple map untyped with a list of maps" {
515	const source =
516	\\a: 0
517	\\b:
518	\\ - foo: 1
519	\\ bar: 2
520	\\ - foo: 3
521	\\ bar: 4
522	\\c: 1
523	;
524
525	var yaml = try Yaml.load(testing.allocator, source);
526	defer yaml.deinit();
527
528	try testing.expectEqual(yaml.docs.items.len, 1);
529
530	const map = yaml.docs.items[0].map;
531	try testing.expect(map.contains("a"));
532	try testing.expect(map.contains("b"));
533	try testing.expect(map.contains("c"));
534	try testing.expectEqual(map.get("a").?.int, 0);
535	try testing.expectEqual(map.get("c").?.int, 1);
536	try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1);
537	try testing.expectEqual(map.get("b").?.list[0].map.get("bar").?.int, 2);
538	try testing.expectEqual(map.get("b").?.list[1].map.get("foo").?.int, 3);
539	try testing.expectEqual(map.get("b").?.list[1].map.get("bar").?.int, 4);
540	}
541
542	test "simple map untyped with a list of maps. no indent" {
543	const source =
544	\\b:
545	\\- foo: 1
546	\\c: 1
547	;
548
549	var yaml = try Yaml.load(testing.allocator, source);
550	defer yaml.deinit();
551
552	try testing.expectEqual(yaml.docs.items.len, 1);
553
554	const map = yaml.docs.items[0].map;
555	try testing.expect(map.contains("b"));
556	try testing.expect(map.contains("c"));
557	try testing.expectEqual(map.get("c").?.int, 1);
558	try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1);
559	}
560
561	test "simple map untyped with a list of maps. no indent 2" {
562	const source =
563	\\a: 0
564	\\b:
565	\\- foo: 1
566	\\ bar: 2
567	\\- foo: 3
568	\\ bar: 4
569	\\c: 1
570	;
571
572	var yaml = try Yaml.load(testing.allocator, source);
573	defer yaml.deinit();
574
575	try testing.expectEqual(yaml.docs.items.len, 1);
576
577	const map = yaml.docs.items[0].map;
578	try testing.expect(map.contains("a"));
579	try testing.expect(map.contains("b"));
580	try testing.expect(map.contains("c"));
581	try testing.expectEqual(map.get("a").?.int, 0);
582	try testing.expectEqual(map.get("c").?.int, 1);
583	try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1);
584	try testing.expectEqual(map.get("b").?.list[0].map.get("bar").?.int, 2);
585	try testing.expectEqual(map.get("b").?.list[1].map.get("foo").?.int, 3);
586	try testing.expectEqual(map.get("b").?.list[1].map.get("bar").?.int, 4);
587	}
588
589	test "simple map typed" {
590	const source =
591	\\a: 0
592	\\b: hello there
593	\\c: 'wait, what?'
594	;
595
596	var yaml = try Yaml.load(testing.allocator, source);
597	defer yaml.deinit();
598
599	const simple = try yaml.parse(struct { a: usize, b: []const u8, c: []const u8 });
600	try testing.expectEqual(simple.a, 0);
601	try testing.expect(mem.eql(u8, simple.b, "hello there"));
602	try testing.expect(mem.eql(u8, simple.c, "wait, what?"));
603	}
604
605	test "typed nested structs" {
606	const source =
607	\\a:
608	\\ b: hello there
609	\\ c: 'wait, what?'
610	;
611
612	var yaml = try Yaml.load(testing.allocator, source);
613	defer yaml.deinit();
614
615	const simple = try yaml.parse(struct {
616	a: struct {
617	b: []const u8,
618	c: []const u8,
619	},
620	});
621	try testing.expect(mem.eql(u8, simple.a.b, "hello there"));
622	try testing.expect(mem.eql(u8, simple.a.c, "wait, what?"));
623	}
624
625	test "multidoc typed as a slice of structs" {
626	const source =
627	\\---
628	\\a: 0
629	\\---
630	\\a: 1
631	\\...
632	;
633
634	var yaml = try Yaml.load(testing.allocator, source);
635	defer yaml.deinit();
636
637	{
638	const result = try yaml.parse([2]struct { a: usize });
639	try testing.expectEqual(result.len, 2);
640	try testing.expectEqual(result[0].a, 0);
641	try testing.expectEqual(result[1].a, 1);
642	}
643
644	{
645	const result = try yaml.parse([]struct { a: usize });
646	try testing.expectEqual(result.len, 2);
647	try testing.expectEqual(result[0].a, 0);
648	try testing.expectEqual(result[1].a, 1);
649	}	487	}
650	}	488	};
651
652	test "multidoc typed as a struct is an error" {
653	const source =
654	\\---
655	\\a: 0
656	\\---
657	\\b: 1
658	\\...
659	;
660
661	var yaml = try Yaml.load(testing.allocator, source);
662	defer yaml.deinit();
663
664	try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { a: usize }));
665	try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { b: usize }));
666	try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { a: usize, b: usize }));
667	}
668
669	test "multidoc typed as a slice of structs with optionals" {
670	const source =
671	\\---
672	\\a: 0
673	\\c: 1.0
674	\\---
675	\\a: 1
676	\\b: different field
677	\\...
678	;
679
680	var yaml = try Yaml.load(testing.allocator, source);
681	defer yaml.deinit();
682
683	const result = try yaml.parse([]struct { a: usize, b: ?[]const u8, c: ?f16 });
684	try testing.expectEqual(result.len, 2);
685
686	try testing.expectEqual(result[0].a, 0);
687	try testing.expect(result[0].b == null);
688	try testing.expect(result[0].c != null);
689	try testing.expectEqual(result[0].c.?, 1.0);
690
691	try testing.expectEqual(result[1].a, 1);
692	try testing.expect(result[1].b != null);
693	try testing.expect(mem.eql(u8, result[1].b.?, "different field"));
694	try testing.expect(result[1].c == null);
695	}
696
697	test "empty yaml can be represented as void" {
698	const source = "";
699	var yaml = try Yaml.load(testing.allocator, source);
700	defer yaml.deinit();
701	const result = try yaml.parse(void);
702	try testing.expect(@TypeOf(result) == void);
703	}
704		489
705	test "nonempty yaml cannot be represented as void" {	490	pub fn stringify(allocator: Allocator, input: anytype, writer: anytype) !void {
706	const source =	491	var arena = ArenaAllocator.init(allocator);
707	\\a: b	492	defer arena.deinit();
708	;
709		493
710	var yaml = try Yaml.load(testing.allocator, source);	494	var maybe_value = try Value.encode(arena.allocator(), input);
711	defer yaml.deinit();
712		495
713	try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(void));	496	if (maybe_value) \|value\| {
		497	// TODO should we output as an explicit doc?
		498	// How can allow the user to specify?
		499	try value.stringify(writer, .{});
		500	}
714	}	501	}
715		502
716	test "typed array size mismatch" {	503	test {
717	const source =	504	std.testing.refAllDecls(Tokenizer);
718	\\- 0	505	std.testing.refAllDecls(parse);
719	\\- 0	506	_ = @import("yaml/test.zig");
720	;
721
722	var yaml = try Yaml.load(testing.allocator, source);
723	defer yaml.deinit();
724
725	try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse([1]usize));
726	try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse([5]usize));
727	}	507	}

src/link/tapi/yaml/test.zig created+475

...	@@ -0,0 +1,475 @@
1	const std = @import("std");
2	const mem = std.mem;
3	const testing = std.testing;
4
5	const yaml_mod = @import("../yaml.zig");
6	const Yaml = yaml_mod.Yaml;
7
8	test "simple list" {
9	const source =
10	\\- a
11	\\- b
12	\\- c
13	;
14
15	var yaml = try Yaml.load(testing.allocator, source);
16	defer yaml.deinit();
17
18	try testing.expectEqual(yaml.docs.items.len, 1);
19
20	const list = yaml.docs.items[0].list;
21	try testing.expectEqual(list.len, 3);
22
23	try testing.expectEqualStrings("a", list[0].string);
24	try testing.expectEqualStrings("b", list[1].string);
25	try testing.expectEqualStrings("c", list[2].string);
26	}
27
28	test "simple list typed as array of strings" {
29	const source =
30	\\- a
31	\\- b
32	\\- c
33	;
34
35	var yaml = try Yaml.load(testing.allocator, source);
36	defer yaml.deinit();
37
38	try testing.expectEqual(yaml.docs.items.len, 1);
39
40	const arr = try yaml.parse([3][]const u8);
41	try testing.expectEqual(3, arr.len);
42	try testing.expectEqualStrings("a", arr[0]);
43	try testing.expectEqualStrings("b", arr[1]);
44	try testing.expectEqualStrings("c", arr[2]);
45	}
46
47	test "simple list typed as array of ints" {
48	const source =
49	\\- 0
50	\\- 1
51	\\- 2
52	;
53
54	var yaml = try Yaml.load(testing.allocator, source);
55	defer yaml.deinit();
56
57	try testing.expectEqual(yaml.docs.items.len, 1);
58
59	const arr = try yaml.parse([3]u8);
60	try testing.expectEqualSlices(u8, &[_]u8{ 0, 1, 2 }, &arr);
61	}
62
63	test "list of mixed sign integer" {
64	const source =
65	\\- 0
66	\\- -1
67	\\- 2
68	;
69
70	var yaml = try Yaml.load(testing.allocator, source);
71	defer yaml.deinit();
72
73	try testing.expectEqual(yaml.docs.items.len, 1);
74
75	const arr = try yaml.parse([3]i8);
76	try testing.expectEqualSlices(i8, &[_]i8{ 0, -1, 2 }, &arr);
77	}
78
79	test "simple map untyped" {
80	const source =
81	\\a: 0
82	;
83
84	var yaml = try Yaml.load(testing.allocator, source);
85	defer yaml.deinit();
86
87	try testing.expectEqual(yaml.docs.items.len, 1);
88
89	const map = yaml.docs.items[0].map;
90	try testing.expect(map.contains("a"));
91	try testing.expectEqual(@as(i64, 0), map.get("a").?.int);
92	}
93
94	test "simple map untyped with a list of maps" {
95	const source =
96	\\a: 0
97	\\b:
98	\\ - foo: 1
99	\\ bar: 2
100	\\ - foo: 3
101	\\ bar: 4
102	\\c: 1
103	;
104
105	var yaml = try Yaml.load(testing.allocator, source);
106	defer yaml.deinit();
107
108	try testing.expectEqual(yaml.docs.items.len, 1);
109
110	const map = yaml.docs.items[0].map;
111	try testing.expect(map.contains("a"));
112	try testing.expect(map.contains("b"));
113	try testing.expect(map.contains("c"));
114	try testing.expectEqual(@as(i64, 0), map.get("a").?.int);
115	try testing.expectEqual(@as(i64, 1), map.get("c").?.int);
116	try testing.expectEqual(@as(i64, 1), map.get("b").?.list[0].map.get("foo").?.int);
117	try testing.expectEqual(@as(i64, 2), map.get("b").?.list[0].map.get("bar").?.int);
118	try testing.expectEqual(@as(i64, 3), map.get("b").?.list[1].map.get("foo").?.int);
119	try testing.expectEqual(@as(i64, 4), map.get("b").?.list[1].map.get("bar").?.int);
120	}
121
122	test "simple map untyped with a list of maps. no indent" {
123	const source =
124	\\b:
125	\\- foo: 1
126	\\c: 1
127	;
128
129	var yaml = try Yaml.load(testing.allocator, source);
130	defer yaml.deinit();
131
132	try testing.expectEqual(yaml.docs.items.len, 1);
133
134	const map = yaml.docs.items[0].map;
135	try testing.expect(map.contains("b"));
136	try testing.expect(map.contains("c"));
137	try testing.expectEqual(@as(i64, 1), map.get("c").?.int);
138	try testing.expectEqual(@as(i64, 1), map.get("b").?.list[0].map.get("foo").?.int);
139	}
140
141	test "simple map untyped with a list of maps. no indent 2" {
142	const source =
143	\\a: 0
144	\\b:
145	\\- foo: 1
146	\\ bar: 2
147	\\- foo: 3
148	\\ bar: 4
149	\\c: 1
150	;
151
152	var yaml = try Yaml.load(testing.allocator, source);
153	defer yaml.deinit();
154
155	try testing.expectEqual(yaml.docs.items.len, 1);
156
157	const map = yaml.docs.items[0].map;
158	try testing.expect(map.contains("a"));
159	try testing.expect(map.contains("b"));
160	try testing.expect(map.contains("c"));
161	try testing.expectEqual(@as(i64, 0), map.get("a").?.int);
162	try testing.expectEqual(@as(i64, 1), map.get("c").?.int);
163	try testing.expectEqual(@as(i64, 1), map.get("b").?.list[0].map.get("foo").?.int);
164	try testing.expectEqual(@as(i64, 2), map.get("b").?.list[0].map.get("bar").?.int);
165	try testing.expectEqual(@as(i64, 3), map.get("b").?.list[1].map.get("foo").?.int);
166	try testing.expectEqual(@as(i64, 4), map.get("b").?.list[1].map.get("bar").?.int);
167	}
168
169	test "simple map typed" {
170	const source =
171	\\a: 0
172	\\b: hello there
173	\\c: 'wait, what?'
174	;
175
176	var yaml = try Yaml.load(testing.allocator, source);
177	defer yaml.deinit();
178
179	const simple = try yaml.parse(struct { a: usize, b: []const u8, c: []const u8 });
180	try testing.expectEqual(@as(usize, 0), simple.a);
181	try testing.expectEqualStrings("hello there", simple.b);
182	try testing.expectEqualStrings("wait, what?", simple.c);
183	}
184
185	test "typed nested structs" {
186	const source =
187	\\a:
188	\\ b: hello there
189	\\ c: 'wait, what?'
190	;
191
192	var yaml = try Yaml.load(testing.allocator, source);
193	defer yaml.deinit();
194
195	const simple = try yaml.parse(struct {
196	a: struct {
197	b: []const u8,
198	c: []const u8,
199	},
200	});
201	try testing.expectEqualStrings("hello there", simple.a.b);
202	try testing.expectEqualStrings("wait, what?", simple.a.c);
203	}
204
205	test "single quoted string" {
206	const source =
207	\\- 'hello'
208	\\- 'here''s an escaped quote'
209	\\- 'newlines and tabs\nare not\tsupported'
210	;
211
212	var yaml = try Yaml.load(testing.allocator, source);
213	defer yaml.deinit();
214
215	const arr = try yaml.parse([3][]const u8);
216	try testing.expectEqual(arr.len, 3);
217	try testing.expectEqualStrings("hello", arr[0]);
218	try testing.expectEqualStrings("here's an escaped quote", arr[1]);
219	try testing.expectEqualStrings("newlines and tabs\\nare not\\tsupported", arr[2]);
220	}
221
222	test "double quoted string" {
223	const source =
224	\\- "hello"
225	\\- "\"here\" are some escaped quotes"
226	\\- "newlines and tabs\nare\tsupported"
227	\\- "let's have
228	\\some fun!"
229	;
230
231	var yaml = try Yaml.load(testing.allocator, source);
232	defer yaml.deinit();
233
234	const arr = try yaml.parse([4][]const u8);
235	try testing.expectEqual(arr.len, 4);
236	try testing.expectEqualStrings("hello", arr[0]);
237	try testing.expectEqualStrings(
238	\\"here" are some escaped quotes
239	, arr[1]);
240	try testing.expectEqualStrings(
241	\\newlines and tabs
242	\\are supported
243	, arr[2]);
244	try testing.expectEqualStrings(
245	\\let's have
246	\\some fun!
247	, arr[3]);
248	}
249
250	test "multidoc typed as a slice of structs" {
251	const source =
252	\\---
253	\\a: 0
254	\\---
255	\\a: 1
256	\\...
257	;
258
259	var yaml = try Yaml.load(testing.allocator, source);
260	defer yaml.deinit();
261
262	{
263	const result = try yaml.parse([2]struct { a: usize });
264	try testing.expectEqual(result.len, 2);
265	try testing.expectEqual(result[0].a, 0);
266	try testing.expectEqual(result[1].a, 1);
267	}
268
269	{
270	const result = try yaml.parse([]struct { a: usize });
271	try testing.expectEqual(result.len, 2);
272	try testing.expectEqual(result[0].a, 0);
273	try testing.expectEqual(result[1].a, 1);
274	}
275	}
276
277	test "multidoc typed as a struct is an error" {
278	const source =
279	\\---
280	\\a: 0
281	\\---
282	\\b: 1
283	\\...
284	;
285
286	var yaml = try Yaml.load(testing.allocator, source);
287	defer yaml.deinit();
288
289	try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { a: usize }));
290	try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { b: usize }));
291	try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(struct { a: usize, b: usize }));
292	}
293
294	test "multidoc typed as a slice of structs with optionals" {
295	const source =
296	\\---
297	\\a: 0
298	\\c: 1.0
299	\\---
300	\\a: 1
301	\\b: different field
302	\\...
303	;
304
305	var yaml = try Yaml.load(testing.allocator, source);
306	defer yaml.deinit();
307
308	const result = try yaml.parse([]struct { a: usize, b: ?[]const u8, c: ?f16 });
309	try testing.expectEqual(result.len, 2);
310
311	try testing.expectEqual(result[0].a, 0);
312	try testing.expect(result[0].b == null);
313	try testing.expect(result[0].c != null);
314	try testing.expectEqual(result[0].c.?, 1.0);
315
316	try testing.expectEqual(result[1].a, 1);
317	try testing.expect(result[1].b != null);
318	try testing.expectEqualStrings("different field", result[1].b.?);
319	try testing.expect(result[1].c == null);
320	}
321
322	test "empty yaml can be represented as void" {
323	const source = "";
324	var yaml = try Yaml.load(testing.allocator, source);
325	defer yaml.deinit();
326	const result = try yaml.parse(void);
327	try testing.expect(@TypeOf(result) == void);
328	}
329
330	test "nonempty yaml cannot be represented as void" {
331	const source =
332	\\a: b
333	;
334
335	var yaml = try Yaml.load(testing.allocator, source);
336	defer yaml.deinit();
337
338	try testing.expectError(Yaml.Error.TypeMismatch, yaml.parse(void));
339	}
340
341	test "typed array size mismatch" {
342	const source =
343	\\- 0
344	\\- 0
345	;
346
347	var yaml = try Yaml.load(testing.allocator, source);
348	defer yaml.deinit();
349
350	try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse([1]usize));
351	try testing.expectError(Yaml.Error.ArraySizeMismatch, yaml.parse([5]usize));
352	}
353
354	test "comments" {
355	const source =
356	\\
357	\\key: # this is the key
358	\\# first value
359	\\
360	\\- val1
361	\\
362	\\# second value
363	\\- val2
364	;
365
366	var yaml = try Yaml.load(testing.allocator, source);
367	defer yaml.deinit();
368
369	const simple = try yaml.parse(struct {
370	key: []const []const u8,
371	});
372	try testing.expect(simple.key.len == 2);
373	try testing.expectEqualStrings("val1", simple.key[0]);
374	try testing.expectEqualStrings("val2", simple.key[1]);
375	}
376
377	test "promote ints to floats in a list mixed numeric types" {
378	const source =
379	\\a_list: [0, 1.0]
380	;
381
382	var yaml = try Yaml.load(testing.allocator, source);
383	defer yaml.deinit();
384
385	const simple = try yaml.parse(struct {
386	a_list: []const f64,
387	});
388	try testing.expectEqualSlices(f64, &[_]f64{ 0.0, 1.0 }, simple.a_list);
389	}
390
391	test "demoting floats to ints in a list is an error" {
392	const source =
393	\\a_list: [0, 1.0]
394	;
395
396	var yaml = try Yaml.load(testing.allocator, source);
397	defer yaml.deinit();
398
399	try testing.expectError(error.TypeMismatch, yaml.parse(struct {
400	a_list: []const u64,
401	}));
402	}
403
404	test "duplicate map keys" {
405	const source =
406	\\a: b
407	\\a: c
408	;
409	try testing.expectError(error.DuplicateMapKey, Yaml.load(testing.allocator, source));
410	}
411
412	fn testStringify(expected: []const u8, input: anytype) !void {
413	var output = std.ArrayList(u8).init(testing.allocator);
414	defer output.deinit();
415
416	try yaml_mod.stringify(testing.allocator, input, output.writer());
417	try testing.expectEqualStrings(expected, output.items);
418	}
419
420	test "stringify an int" {
421	try testStringify("128", @as(u32, 128));
422	}
423
424	test "stringify a simple struct" {
425	try testStringify(
426	\\a: 1
427	\\b: 2
428	\\c: 2.5
429	, struct { a: i64, b: f64, c: f64 }{ .a = 1, .b = 2.0, .c = 2.5 });
430	}
431
432	test "stringify a struct with an optional" {
433	try testStringify(
434	\\a: 1
435	\\b: 2
436	\\c: 2.5
437	, struct { a: i64, b: ?f64, c: f64 }{ .a = 1, .b = 2.0, .c = 2.5 });
438
439	try testStringify(
440	\\a: 1
441	\\c: 2.5
442	, struct { a: i64, b: ?f64, c: f64 }{ .a = 1, .b = null, .c = 2.5 });
443	}
444
445	test "stringify a struct with all optionals" {
446	try testStringify("", struct { a: ?i64, b: ?f64 }{ .a = null, .b = null });
447	}
448
449	test "stringify an optional" {
450	try testStringify("", null);
451	try testStringify("", @as(?u64, null));
452	}
453
454	test "stringify a union" {
455	const Dummy = union(enum) {
456	x: u64,
457	y: f64,
458	};
459	try testStringify("a: 1", struct { a: Dummy }{ .a = .{ .x = 1 } });
460	try testStringify("a: 2.1", struct { a: Dummy }{ .a = .{ .y = 2.1 } });
461	}
462
463	test "stringify a string" {
464	try testStringify("a: name", struct { a: []const u8 }{ .a = "name" });
465	try testStringify("name", "name");
466	}
467
468	test "stringify a list" {
469	try testStringify("[ 1, 2, 3 ]", @as([]const u64, &.{ 1, 2, 3 }));
470	try testStringify("[ 1, 2, 3 ]", .{ @as(i64, 1), 2, 3 });
471	try testStringify("[ 1, name, 3 ]", .{ 1, "name", 3 });
472
473	const arr: [3]i64 = .{ 1, 2, 3 };
474	try testStringify("[ 1, 2, 3 ]", arr);
475	}