authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-06-01 23:44:23+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-06-02 10:41:13+02:00
loge498fb155051f548071da1a13098b8793f527275
tree5de7394cce91394c83beeeeb9486e81a37650aa0
parent288e89b606b46328a5ab358b2eef2c5dc277bc8f

tapi: sync with upstream

gitrev kubkon/zig-yaml 8cf8dc3bb901fac8189f441392fc0989ad14cf71 Calculate line and col info indexed by token index. We can then re-use this info to track current column number (aka indentation level) of each "key:value" pair (map) or "- element" (list). This significantly cleans up the code, and leads naturally to handling of unindented lists in tbd files.

4 files changed, 190 insertions(+), 110 deletions(-)

src/link/tapi/Tokenizer.zig+39-18
......@@ -11,9 +11,6 @@ pub const Token = struct {
1111 id: Id,
1212 start: usize,
1313 end: usize,
14 // Count of spaces/tabs.
15 // Only active for .Space and .Tab tokens.
16 count: ?usize = null,
1714
1815 pub const Id = enum {
1916 Eof,
......@@ -87,8 +84,8 @@ pub fn next(self: *Tokenizer) Token {
8784 var state: union(enum) {
8885 Start,
8986 NewLine,
90 Space: usize,
91 Tab: usize,
87 Space,
88 Tab,
9289 Hyphen: usize,
9390 Dot: usize,
9491 Literal,
......@@ -99,10 +96,10 @@ pub fn next(self: *Tokenizer) Token {
9996 switch (state) {
10097 .Start => switch (c) {
10198 ' ' => {
102 state = .{ .Space = 1 };
99 state = .Space;
103100 },
104101 '\t' => {
105 state = .{ .Tab = 1 };
102 state = .Tab;
106103 },
107104 '\n' => {
108105 result.id = .NewLine;
......@@ -182,23 +179,17 @@ pub fn next(self: *Tokenizer) Token {
182179 state = .Literal;
183180 },
184181 },
185 .Space => |*count| switch (c) {
186 ' ' => {
187 count.* += 1;
188 },
182 .Space => switch (c) {
183 ' ' => {},
189184 else => {
190185 result.id = .Space;
191 result.count = count.*;
192186 break;
193187 },
194188 },
195 .Tab => |*count| switch (c) {
196 ' ' => {
197 count.* += 1;
198 },
189 .Tab => switch (c) {
190 '\t' => {},
199191 else => {
200192 result.id = .Tab;
201 result.count = count.*;
202193 break;
203194 },
204195 },
......@@ -272,10 +263,18 @@ fn testExpected(source: []const u8, expected: []const Token.Id) !void {
272263 .buffer = source,
273264 };
274265
266 var token_len: usize = 0;
275267 for (expected) |exp| {
268 token_len += 1;
276269 const token = tokenizer.next();
277270 try testing.expectEqual(exp, token.id);
278271 }
272
273 while (tokenizer.next().id != .Eof) {
274 token_len += 1; // consume all tokens
275 }
276
277 try testing.expectEqual(expected.len, token_len);
279278}
280279
281280test "empty doc" {
......@@ -376,7 +375,7 @@ test "inline mapped sequence of values" {
376375 });
377376}
378377
379test "part of tdb" {
378test "part of tbd" {
380379 try testExpected(
381380 \\--- !tapi-tbd
382381 \\tbd-version: 4
......@@ -437,3 +436,25 @@ test "part of tdb" {
437436 .Eof,
438437 });
439438}
439
440test "Unindented list" {
441 try testExpected(
442 \\b:
443 \\- foo: 1
444 \\c: 1
445 , &[_]Token.Id{
446 .Literal,
447 .MapValueInd,
448 .NewLine,
449 .SeqItemInd,
450 .Literal,
451 .MapValueInd,
452 .Space,
453 .Literal,
454 .NewLine,
455 .Literal,
456 .MapValueInd,
457 .Space,
458 .Literal,
459 });
460}
src/link/tapi/parse.zig+72-90
......@@ -42,7 +42,7 @@ pub const Node = struct {
4242 .doc => @fieldParentPtr(Node.Doc, "base", self).deinit(allocator),
4343 .map => @fieldParentPtr(Node.Map, "base", self).deinit(allocator),
4444 .list => @fieldParentPtr(Node.List, "base", self).deinit(allocator),
45 .value => {},
45 .value => @fieldParentPtr(Node.Value, "base", self).deinit(allocator),
4646 }
4747 }
4848
......@@ -82,8 +82,8 @@ pub const Node = struct {
8282 options: std.fmt.FormatOptions,
8383 writer: anytype,
8484 ) !void {
85 _ = fmt;
8685 _ = options;
86 _ = fmt;
8787 if (self.directive) |id| {
8888 try std.fmt.format(writer, "{{ ", .{});
8989 const directive = self.base.tree.tokens[id];
......@@ -127,8 +127,8 @@ pub const Node = struct {
127127 options: std.fmt.FormatOptions,
128128 writer: anytype,
129129 ) !void {
130 _ = fmt;
131130 _ = options;
131 _ = fmt;
132132 try std.fmt.format(writer, "{{ ", .{});
133133 for (self.values.items) |entry| {
134134 const key = self.base.tree.tokens[entry.key];
......@@ -163,8 +163,8 @@ pub const Node = struct {
163163 options: std.fmt.FormatOptions,
164164 writer: anytype,
165165 ) !void {
166 _ = fmt;
167166 _ = options;
167 _ = fmt;
168168 try std.fmt.format(writer, "[ ", .{});
169169 for (self.values.items) |node| {
170170 try std.fmt.format(writer, "{}, ", .{node});
......@@ -180,14 +180,19 @@ pub const Node = struct {
180180
181181 pub const base_tag: Node.Tag = .value;
182182
183 pub fn deinit(self: *Value, allocator: Allocator) void {
184 _ = self;
185 _ = allocator;
186 }
187
183188 pub fn format(
184189 self: *const Value,
185190 comptime fmt: []const u8,
186191 options: std.fmt.FormatOptions,
187192 writer: anytype,
188193 ) !void {
189 _ = fmt;
190194 _ = options;
195 _ = fmt;
191196 const start = self.base.tree.tokens[self.start.?];
192197 const end = self.base.tree.tokens[self.end.?];
193198 return std.fmt.format(writer, "{s}", .{
......@@ -197,10 +202,16 @@ pub const Node = struct {
197202 };
198203};
199204
205pub const LineCol = struct {
206 line: usize,
207 col: usize,
208};
209
200210pub const Tree = struct {
201211 allocator: Allocator,
202212 source: []const u8,
203213 tokens: []Token,
214 line_cols: std.AutoHashMap(TokenIndex, LineCol),
204215 docs: std.ArrayListUnmanaged(*Node) = .{},
205216
206217 pub fn init(allocator: Allocator) Tree {
......@@ -208,11 +219,13 @@ pub const Tree = struct {
208219 .allocator = allocator,
209220 .source = undefined,
210221 .tokens = undefined,
222 .line_cols = std.AutoHashMap(TokenIndex, LineCol).init(allocator),
211223 };
212224 }
213225
214226 pub fn deinit(self: *Tree) void {
215227 self.allocator.free(self.tokens);
228 self.line_cols.deinit();
216229 for (self.docs.items) |doc| {
217230 doc.deinit(self.allocator);
218231 self.allocator.destroy(doc);
......@@ -223,12 +236,29 @@ pub const Tree = struct {
223236 pub fn parse(self: *Tree, source: []const u8) !void {
224237 var tokenizer = Tokenizer{ .buffer = source };
225238 var tokens = std.ArrayList(Token).init(self.allocator);
226 errdefer tokens.deinit();
239 defer tokens.deinit();
240
241 var line: usize = 0;
242 var prev_line_last_col: usize = 0;
227243
228244 while (true) {
229245 const token = tokenizer.next();
246 const tok_id = tokens.items.len;
230247 try tokens.append(token);
231 if (token.id == .Eof) break;
248
249 try self.line_cols.putNoClobber(tok_id, .{
250 .line = line,
251 .col = token.start - prev_line_last_col,
252 });
253
254 switch (token.id) {
255 .Eof => break,
256 .NewLine => {
257 line += 1;
258 prev_line_last_col = token.end;
259 },
260 else => {},
261 }
232262 }
233263
234264 self.source = source;
......@@ -239,15 +269,12 @@ pub const Tree = struct {
239269 .allocator = self.allocator,
240270 .tree = self,
241271 .token_it = &it,
272 .line_cols = &self.line_cols,
242273 };
243 defer parser.deinit();
244
245 try parser.scopes.append(self.allocator, .{
246 .indent = 0,
247 });
248274
249275 while (true) {
250276 if (parser.token_it.peek() == null) return;
277
251278 const pos = parser.token_it.pos;
252279 const token = parser.token_it.next();
253280
......@@ -269,22 +296,12 @@ const Parser = struct {
269296 allocator: Allocator,
270297 tree: *Tree,
271298 token_it: *TokenIterator,
272 scopes: std.ArrayListUnmanaged(Scope) = .{},
273
274 const Scope = struct {
275 indent: usize,
276 };
277
278 fn deinit(self: *Parser) void {
279 self.scopes.deinit(self.allocator);
280 }
299 line_cols: *const std.AutoHashMap(TokenIndex, LineCol),
281300
282301 fn doc(self: *Parser, start: TokenIndex) ParseError!*Node.Doc {
283302 const node = try self.allocator.create(Node.Doc);
284303 errdefer self.allocator.destroy(node);
285 node.* = .{
286 .start = start,
287 };
304 node.* = .{ .start = start };
288305 node.base.tree = self.tree;
289306
290307 self.token_it.seekTo(start);
......@@ -346,19 +363,24 @@ const Parser = struct {
346363 fn map(self: *Parser, start: TokenIndex) ParseError!*Node.Map {
347364 const node = try self.allocator.create(Node.Map);
348365 errdefer self.allocator.destroy(node);
349 node.* = .{
350 .start = start,
351 };
366 node.* = .{ .start = start };
352367 node.base.tree = self.tree;
353368
354369 self.token_it.seekTo(start);
355370
356371 log.debug("Map start: {}, {}", .{ start, self.tree.tokens[start] });
357 log.debug("Current scope: {}", .{self.scopes.items[self.scopes.items.len - 1]});
372
373 const col = self.getCol(start);
358374
359375 while (true) {
376 self.eatCommentsAndSpace();
377
360378 // Parse key.
361379 const key_pos = self.token_it.pos;
380 if (self.getCol(key_pos) != col) {
381 break;
382 }
383
362384 const key = self.token_it.next();
363385 switch (key.id) {
364386 .Literal => {},
......@@ -372,13 +394,13 @@ const Parser = struct {
372394
373395 // Separator
374396 _ = try self.expectToken(.MapValueInd);
375 self.eatCommentsAndSpace();
376397
377398 // Parse value.
378399 const value: *Node = value: {
379400 if (self.eatToken(.NewLine)) |_| {
401 self.eatCommentsAndSpace();
402
380403 // Explicit, complex value such as list or map.
381 try self.openScope();
382404 const value_pos = self.token_it.pos;
383405 const value = self.token_it.next();
384406 switch (value.id) {
......@@ -398,6 +420,8 @@ const Parser = struct {
398420 },
399421 }
400422 } else {
423 self.eatCommentsAndSpace();
424
401425 const value_pos = self.token_it.pos;
402426 const value = self.token_it.next();
403427 switch (value.id) {
......@@ -424,11 +448,7 @@ const Parser = struct {
424448 .value = value,
425449 });
426450
427 if (self.eatToken(.NewLine)) |_| {
428 if (try self.closeScope()) {
429 break;
430 }
431 }
451 _ = self.eatToken(.NewLine);
432452 }
433453
434454 node.end = self.token_it.pos - 1;
......@@ -449,14 +469,18 @@ const Parser = struct {
449469 self.token_it.seekTo(start);
450470
451471 log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] });
452 log.debug("Current scope: {}", .{self.scopes.items[self.scopes.items.len - 1]});
472
473 const col = self.getCol(start);
453474
454475 while (true) {
476 self.eatCommentsAndSpace();
477
478 if (self.getCol(self.token_it.pos) != col) {
479 break;
480 }
455481 _ = self.eatToken(.SeqItemInd) orelse {
456 _ = try self.closeScope();
457482 break;
458483 };
459 self.eatCommentsAndSpace();
460484
461485 const pos = self.token_it.pos;
462486 const token = self.token_it.next();
......@@ -464,9 +488,6 @@ const Parser = struct {
464488 switch (token.id) {
465489 .Literal, .SingleQuote, .DoubleQuote => {
466490 if (self.eatToken(.MapValueInd)) |_| {
467 if (self.eatToken(.NewLine)) |_| {
468 try self.openScope();
469 }
470491 // nested map
471492 const map_node = try self.map(pos);
472493 break :value &map_node.base;
......@@ -501,15 +522,12 @@ const Parser = struct {
501522 fn list_bracketed(self: *Parser, start: TokenIndex) ParseError!*Node.List {
502523 const node = try self.allocator.create(Node.List);
503524 errdefer self.allocator.destroy(node);
504 node.* = .{
505 .start = start,
506 };
525 node.* = .{ .start = start };
507526 node.base.tree = self.tree;
508527
509528 self.token_it.seekTo(start);
510529
511530 log.debug("List start: {}, {}", .{ start, self.tree.tokens[start] });
512 log.debug("Current scope: {}", .{self.scopes.items[self.scopes.items.len - 1]});
513531
514532 _ = try self.expectToken(.FlowSeqStart);
515533
......@@ -556,9 +574,7 @@ const Parser = struct {
556574 fn leaf_value(self: *Parser, start: TokenIndex) ParseError!*Node.Value {
557575 const node = try self.allocator.create(Node.Value);
558576 errdefer self.allocator.destroy(node);
559 node.* = .{
560 .start = start,
561 };
577 node.* = .{ .start = start };
562578 node.base.tree = self.tree;
563579
564580 self.token_it.seekTo(start);
......@@ -625,48 +641,6 @@ const Parser = struct {
625641 return node;
626642 }
627643
628 fn openScope(self: *Parser) !void {
629 const peek = self.token_it.peek() orelse return error.UnexpectedEof;
630 if (peek.id != .Space and peek.id != .Tab) {
631 // No need to open scope.
632 return;
633 }
634 const indent = self.token_it.next().count.?;
635 const prev_scope = self.scopes.items[self.scopes.items.len - 1];
636 if (indent < prev_scope.indent) {
637 return error.MalformedYaml;
638 }
639
640 log.debug("Opening scope...", .{});
641
642 try self.scopes.append(self.allocator, .{
643 .indent = indent,
644 });
645 }
646
647 fn closeScope(self: *Parser) !bool {
648 const indent = indent: {
649 const peek = self.token_it.peek() orelse return error.UnexpectedEof;
650 switch (peek.id) {
651 .Space, .Tab => {
652 break :indent self.token_it.next().count.?;
653 },
654 else => {
655 break :indent 0;
656 },
657 }
658 };
659
660 const scope = self.scopes.items[self.scopes.items.len - 1];
661 if (indent < scope.indent) {
662 log.debug("Closing scope...", .{});
663 _ = self.scopes.pop();
664 return true;
665 }
666
667 return false;
668 }
669
670644 fn eatCommentsAndSpace(self: *Parser) void {
671645 while (true) {
672646 _ = self.token_it.peek() orelse return;
......@@ -701,6 +675,14 @@ const Parser = struct {
701675 fn expectToken(self: *Parser, id: Token.Id) ParseError!TokenIndex {
702676 return self.eatToken(id) orelse error.UnexpectedToken;
703677 }
678
679 fn getLine(self: *Parser, index: TokenIndex) usize {
680 return self.line_cols.get(index).?.line;
681 }
682
683 fn getCol(self: *Parser, index: TokenIndex) usize {
684 return self.line_cols.get(index).?.col;
685 }
704686};
705687
706688test {
src/link/tapi/parse/test.zig+4-2
......@@ -1,8 +1,10 @@
11const std = @import("std");
22const mem = std.mem;
33const testing = std.testing;
4const Tree = @import("../parse.zig").Tree;
5const Node = @import("../parse.zig").Node;
4const parse = @import("../parse.zig");
5
6const Node = parse.Node;
7const Tree = parse.Tree;
68
79test "explicit doc" {
810 const source =
src/link/tapi/yaml.zig+75
......@@ -511,6 +511,81 @@ test "simple map untyped" {
511511 try testing.expectEqual(map.get("a").?.int, 0);
512512}
513513
514test "simple map untyped with a list of maps" {
515 const source =
516 \\a: 0
517 \\b:
518 \\ - foo: 1
519 \\ bar: 2
520 \\ - foo: 3
521 \\ bar: 4
522 \\c: 1
523 ;
524
525 var yaml = try Yaml.load(testing.allocator, source);
526 defer yaml.deinit();
527
528 try testing.expectEqual(yaml.docs.items.len, 1);
529
530 const map = yaml.docs.items[0].map;
531 try testing.expect(map.contains("a"));
532 try testing.expect(map.contains("b"));
533 try testing.expect(map.contains("c"));
534 try testing.expectEqual(map.get("a").?.int, 0);
535 try testing.expectEqual(map.get("c").?.int, 1);
536 try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1);
537 try testing.expectEqual(map.get("b").?.list[0].map.get("bar").?.int, 2);
538 try testing.expectEqual(map.get("b").?.list[1].map.get("foo").?.int, 3);
539 try testing.expectEqual(map.get("b").?.list[1].map.get("bar").?.int, 4);
540}
541
542test "simple map untyped with a list of maps. no indent" {
543 const source =
544 \\b:
545 \\- foo: 1
546 \\c: 1
547 ;
548
549 var yaml = try Yaml.load(testing.allocator, source);
550 defer yaml.deinit();
551
552 try testing.expectEqual(yaml.docs.items.len, 1);
553
554 const map = yaml.docs.items[0].map;
555 try testing.expect(map.contains("b"));
556 try testing.expect(map.contains("c"));
557 try testing.expectEqual(map.get("c").?.int, 1);
558 try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1);
559}
560
561test "simple map untyped with a list of maps. no indent 2" {
562 const source =
563 \\a: 0
564 \\b:
565 \\- foo: 1
566 \\ bar: 2
567 \\- foo: 3
568 \\ bar: 4
569 \\c: 1
570 ;
571
572 var yaml = try Yaml.load(testing.allocator, source);
573 defer yaml.deinit();
574
575 try testing.expectEqual(yaml.docs.items.len, 1);
576
577 const map = yaml.docs.items[0].map;
578 try testing.expect(map.contains("a"));
579 try testing.expect(map.contains("b"));
580 try testing.expect(map.contains("c"));
581 try testing.expectEqual(map.get("a").?.int, 0);
582 try testing.expectEqual(map.get("c").?.int, 1);
583 try testing.expectEqual(map.get("b").?.list[0].map.get("foo").?.int, 1);
584 try testing.expectEqual(map.get("b").?.list[0].map.get("bar").?.int, 2);
585 try testing.expectEqual(map.get("b").?.list[1].map.get("foo").?.int, 3);
586 try testing.expectEqual(map.get("b").?.list[1].map.get("bar").?.int, 4);
587}
588
514589test "simple map typed" {
515590 const source =
516591 \\a: 0