authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-09-19 14:08:32+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 15:23:41-04:00
logb759308fb30b130ea5d084c3814b1820a589ceee
treecfa5c3e41b86c2be17b0b40739f4ef3c3880a8fe
parentf27bc79121bd9971c9e3465fdba3cc7f7d6ff864

stage2: DepTokenizer print errors

1 files changed, 78 insertions(+), 29 deletions(-)

src-self-hosted/DepTokenizer.zig+78-29
......@@ -228,7 +228,7 @@ pub fn next(self: *Tokenizer) ?Token {
228228 .rhs_continuation_linefeed,
229229 => return null,
230230 .target => {
231 return Token{ .incomplete_target = self.bytes[start..] };
231 return errorPosition(.incomplete_target, start, self.bytes[start..]);
232232 },
233233 .target_reverse_solidus,
234234 .target_dollar_sign,
......@@ -259,7 +259,7 @@ pub fn next(self: *Tokenizer) ?Token {
259259 return null;
260260 },
261261 .prereq_quote => {
262 return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] };
262 return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
263263 },
264264 .prereq => {
265265 self.state = .lhs;
......@@ -278,6 +278,10 @@ pub fn next(self: *Tokenizer) ?Token {
278278 unreachable;
279279}
280280
281fn errorPosition(comptime id: @TagType(Token), index: usize, bytes: []const u8) Token {
282 return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes });
283}
284
281285fn errorIllegalChar(comptime id: @TagType(Token), index: usize, char: u8) Token {
282286 return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
283287}
......@@ -309,8 +313,10 @@ pub const Token = union(enum) {
309313 target: []const u8,
310314 target_must_resolve: []const u8,
311315 prereq: []const u8,
312 incomplete_quoted_prerequisite: []const u8,
313 incomplete_target: []const u8,
316
317 incomplete_quoted_prerequisite: IndexAndBytes,
318 incomplete_target: IndexAndBytes,
319
314320 invalid_target: IndexAndChar,
315321 bad_target_escape: IndexAndChar,
316322 expected_dollar_sign: IndexAndChar,
......@@ -322,11 +328,15 @@ pub const Token = union(enum) {
322328 char: u8,
323329 };
324330
331 pub const IndexAndBytes = struct {
332 index: usize,
333 bytes: []const u8,
334 };
335
325336 /// Resolve escapes in target. Only valid with .target_must_resolve.
326 pub fn resolve(self: Token, buf: *std.ArrayList(u8)) std.mem.Allocator.Error!void {
337 pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
327338 const bytes = self.target_must_resolve; // resolve called on incorrect token
328339
329 try buf.ensureCapacity(bytes.len); // cannot be longer than the unescaped string
330340 var state: enum { start, escape, dollar } = .start;
331341 for (bytes) |c| {
332342 switch (state) {
......@@ -334,33 +344,74 @@ pub const Token = union(enum) {
334344 switch (c) {
335345 '\\' => state = .escape,
336346 '$' => state = .dollar,
337 else => buf.appendAssumeCapacity(c),
347 else => try writer.writeByte(c),
338348 }
339349 },
340350 .escape => {
341351 switch (c) {
342352 ' ', '#', '\\' => {},
343353 '$' => {
344 buf.appendAssumeCapacity('\\');
354 try writer.writeByte('\\');
345355 state = .dollar;
346356 continue;
347357 },
348 else => buf.appendAssumeCapacity('\\'),
358 else => try writer.writeByte('\\'),
349359 }
350 buf.appendAssumeCapacity(c);
360 try writer.writeByte(c);
351361 state = .start;
352362 },
353363 .dollar => {
354 buf.appendAssumeCapacity('$');
364 try writer.writeByte('$');
355365 switch (c) {
356366 '$' => {},
357 else => buf.appendAssumeCapacity(c),
367 else => try writer.writeByte(c),
358368 }
359369 state = .start;
360370 },
361371 }
362372 }
363373 }
374
375 pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
376 switch (self) {
377 .target, .target_must_resolve, .prereq => unreachable, // not an error
378 .incomplete_quoted_prerequisite,
379 .incomplete_target,
380 => |index_and_bytes| {
381 try writer.print("{} '", .{self.errStr()});
382 if (self == .incomplete_target) {
383 const tmp = Token{ .target_must_resolve = index_and_bytes.bytes };
384 try tmp.resolve(writer);
385 } else {
386 try printCharValues(writer, index_and_bytes.bytes);
387 }
388 try writer.print("' at position {}", .{index_and_bytes.index});
389 },
390 .invalid_target,
391 .bad_target_escape,
392 .expected_dollar_sign,
393 .continuation_eol,
394 .incomplete_escape,
395 => |index_and_char| {
396 try writer.writeAll("illegal char ");
397 try printUnderstandableChar(writer, index_and_char.char);
398 try writer.print(" at position {}: {}", .{ index_and_char.index, self.errStr() });
399 },
400 }
401 }
402
403 fn errStr(self: Token) []const u8 {
404 return switch (self) {
405 .target, .target_must_resolve, .prereq => unreachable, // not an error
406 .incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
407 .incomplete_target => "incomplete target",
408 .invalid_target => "invalid target",
409 .bad_target_escape => "bad target escape",
410 .expected_dollar_sign => "expecting '$'",
411 .continuation_eol => "continuation expecting end-of-line",
412 .incomplete_escape => "incomplete escape",
413 };
414 }
364415};
365416
366417test "empty file" {
......@@ -755,16 +806,16 @@ test "error incomplete target" {
755806 );
756807
757808 try depTokenizer("\\ foo.o",
758 \\ERROR: incomplete target ' foo.o' at position 1
809 \\ERROR: incomplete target ' foo.o' at position 0
759810 );
760811 try depTokenizer("\\#foo.o",
761 \\ERROR: incomplete target '#foo.o' at position 1
812 \\ERROR: incomplete target '#foo.o' at position 0
762813 );
763814 try depTokenizer("\\\\foo.o",
764 \\ERROR: incomplete target '\foo.o' at position 1
815 \\ERROR: incomplete target '\foo.o' at position 0
765816 );
766817 try depTokenizer("$$foo.o",
767 \\ERROR: incomplete target '$foo.o' at position 1
818 \\ERROR: incomplete target '$foo.o' at position 0
768819 );
769820}
770821
......@@ -862,7 +913,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
862913 },
863914 .target_must_resolve => {
864915 try buffer.appendSlice("target = {");
865 try token.resolve(&resolve_buf);
916 try token.resolve(resolve_buf.writer());
866917 for (resolve_buf.items) |b| {
867918 try buffer.append(printable_char_tab[b]);
868919 }
......@@ -870,7 +921,9 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
870921 try buffer.appendSlice("}");
871922 },
872923 else => {
873 @panic("TODO");
924 try buffer.appendSlice("ERROR: ");
925 try token.printError(buffer.outStream());
926 break;
874927 },
875928 }
876929 i += 1;
......@@ -1005,23 +1058,19 @@ fn printCharValues(out: anytype, bytes: []const u8) !void {
10051058 }
10061059}
10071060
1008fn printUnderstandableChar(buffer: *std.ArrayListSentineled(u8, 0), char: u8) !void {
1061fn printUnderstandableChar(out: anytype, char: u8) !void {
10091062 if (!std.ascii.isPrint(char) or char == ' ') {
1010 try buffer.outStream().print("\\x{X:0>2}", .{char});
1063 try out.print("\\x{X:0>2}", .{char});
10111064 } else {
1012 try buffer.appendSlice("'");
1013 try buffer.append(printable_char_tab[char]);
1014 try buffer.appendSlice("'");
1065 try out.print("'{c}'", .{printable_char_tab[char]});
10151066 }
10161067}
10171068
10181069// zig fmt: off
1019const printable_char_tab: []const u8 =
1070const printable_char_tab: [256]u8 = (
10201071 "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
10211072 "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++
10221073 "................................................................" ++
1023 "................................................................";
1024// zig fmt: on
1025comptime {
1026 assert(printable_char_tab.len == 256);
1027}
1074 "................................................................"
1075).*;
1076