authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-09-19 14:08:32+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 15:23:41-04:00
logb759308fb30b130ea5d084c3814b1820a589ceee
treecfa5c3e41b86c2be17b0b40739f4ef3c3880a8fe
parentf27bc79121bd9971c9e3465fdba3cc7f7d6ff864

stage2: DepTokenizer print errors

1 files changed, 78 insertions(+), 29 deletions(-)

src-self-hosted/DepTokenizer.zig+78-29
...@@ -228,7 +228,7 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -228,7 +228,7 @@ pub fn next(self: *Tokenizer) ?Token {
228 .rhs_continuation_linefeed,228 .rhs_continuation_linefeed,
229 => return null,229 => return null,
230 .target => {230 .target => {
231 return Token{ .incomplete_target = self.bytes[start..] };231 return errorPosition(.incomplete_target, start, self.bytes[start..]);
232 },232 },
233 .target_reverse_solidus,233 .target_reverse_solidus,
234 .target_dollar_sign,234 .target_dollar_sign,
...@@ -259,7 +259,7 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -259,7 +259,7 @@ pub fn next(self: *Tokenizer) ?Token {
259 return null;259 return null;
260 },260 },
261 .prereq_quote => {261 .prereq_quote => {
262 return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] };262 return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
263 },263 },
264 .prereq => {264 .prereq => {
265 self.state = .lhs;265 self.state = .lhs;
...@@ -278,6 +278,10 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -278,6 +278,10 @@ pub fn next(self: *Tokenizer) ?Token {
278 unreachable;278 unreachable;
279}279}
280280
281fn errorPosition(comptime id: @TagType(Token), index: usize, bytes: []const u8) Token {
282 return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes });
283}
284
281fn errorIllegalChar(comptime id: @TagType(Token), index: usize, char: u8) Token {285fn errorIllegalChar(comptime id: @TagType(Token), index: usize, char: u8) Token {
282 return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });286 return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
283}287}
...@@ -309,8 +313,10 @@ pub const Token = union(enum) {...@@ -309,8 +313,10 @@ pub const Token = union(enum) {
309 target: []const u8,313 target: []const u8,
310 target_must_resolve: []const u8,314 target_must_resolve: []const u8,
311 prereq: []const u8,315 prereq: []const u8,
312 incomplete_quoted_prerequisite: []const u8,316
313 incomplete_target: []const u8,317 incomplete_quoted_prerequisite: IndexAndBytes,
318 incomplete_target: IndexAndBytes,
319
314 invalid_target: IndexAndChar,320 invalid_target: IndexAndChar,
315 bad_target_escape: IndexAndChar,321 bad_target_escape: IndexAndChar,
316 expected_dollar_sign: IndexAndChar,322 expected_dollar_sign: IndexAndChar,
...@@ -322,11 +328,15 @@ pub const Token = union(enum) {...@@ -322,11 +328,15 @@ pub const Token = union(enum) {
322 char: u8,328 char: u8,
323 };329 };
324330
331 pub const IndexAndBytes = struct {
332 index: usize,
333 bytes: []const u8,
334 };
335
325 /// Resolve escapes in target. Only valid with .target_must_resolve.336 /// Resolve escapes in target. Only valid with .target_must_resolve.
326 pub fn resolve(self: Token, buf: *std.ArrayList(u8)) std.mem.Allocator.Error!void {337 pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
327 const bytes = self.target_must_resolve; // resolve called on incorrect token338 const bytes = self.target_must_resolve; // resolve called on incorrect token
328339
329 try buf.ensureCapacity(bytes.len); // cannot be longer than the unescaped string
330 var state: enum { start, escape, dollar } = .start;340 var state: enum { start, escape, dollar } = .start;
331 for (bytes) |c| {341 for (bytes) |c| {
332 switch (state) {342 switch (state) {
...@@ -334,33 +344,74 @@ pub const Token = union(enum) {...@@ -334,33 +344,74 @@ pub const Token = union(enum) {
334 switch (c) {344 switch (c) {
335 '\\' => state = .escape,345 '\\' => state = .escape,
336 '$' => state = .dollar,346 '$' => state = .dollar,
337 else => buf.appendAssumeCapacity(c),347 else => try writer.writeByte(c),
338 }348 }
339 },349 },
340 .escape => {350 .escape => {
341 switch (c) {351 switch (c) {
342 ' ', '#', '\\' => {},352 ' ', '#', '\\' => {},
343 '$' => {353 '$' => {
344 buf.appendAssumeCapacity('\\');354 try writer.writeByte('\\');
345 state = .dollar;355 state = .dollar;
346 continue;356 continue;
347 },357 },
348 else => buf.appendAssumeCapacity('\\'),358 else => try writer.writeByte('\\'),
349 }359 }
350 buf.appendAssumeCapacity(c);360 try writer.writeByte(c);
351 state = .start;361 state = .start;
352 },362 },
353 .dollar => {363 .dollar => {
354 buf.appendAssumeCapacity('$');364 try writer.writeByte('$');
355 switch (c) {365 switch (c) {
356 '$' => {},366 '$' => {},
357 else => buf.appendAssumeCapacity(c),367 else => try writer.writeByte(c),
358 }368 }
359 state = .start;369 state = .start;
360 },370 },
361 }371 }
362 }372 }
363 }373 }
374
375 pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
376 switch (self) {
377 .target, .target_must_resolve, .prereq => unreachable, // not an error
378 .incomplete_quoted_prerequisite,
379 .incomplete_target,
380 => |index_and_bytes| {
381 try writer.print("{} '", .{self.errStr()});
382 if (self == .incomplete_target) {
383 const tmp = Token{ .target_must_resolve = index_and_bytes.bytes };
384 try tmp.resolve(writer);
385 } else {
386 try printCharValues(writer, index_and_bytes.bytes);
387 }
388 try writer.print("' at position {}", .{index_and_bytes.index});
389 },
390 .invalid_target,
391 .bad_target_escape,
392 .expected_dollar_sign,
393 .continuation_eol,
394 .incomplete_escape,
395 => |index_and_char| {
396 try writer.writeAll("illegal char ");
397 try printUnderstandableChar(writer, index_and_char.char);
398 try writer.print(" at position {}: {}", .{ index_and_char.index, self.errStr() });
399 },
400 }
401 }
402
403 fn errStr(self: Token) []const u8 {
404 return switch (self) {
405 .target, .target_must_resolve, .prereq => unreachable, // not an error
406 .incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
407 .incomplete_target => "incomplete target",
408 .invalid_target => "invalid target",
409 .bad_target_escape => "bad target escape",
410 .expected_dollar_sign => "expecting '$'",
411 .continuation_eol => "continuation expecting end-of-line",
412 .incomplete_escape => "incomplete escape",
413 };
414 }
364};415};
365416
366test "empty file" {417test "empty file" {
...@@ -755,16 +806,16 @@ test "error incomplete target" {...@@ -755,16 +806,16 @@ test "error incomplete target" {
755 );806 );
756807
757 try depTokenizer("\\ foo.o",808 try depTokenizer("\\ foo.o",
758 \\ERROR: incomplete target ' foo.o' at position 1809 \\ERROR: incomplete target ' foo.o' at position 0
759 );810 );
760 try depTokenizer("\\#foo.o",811 try depTokenizer("\\#foo.o",
761 \\ERROR: incomplete target '#foo.o' at position 1812 \\ERROR: incomplete target '#foo.o' at position 0
762 );813 );
763 try depTokenizer("\\\\foo.o",814 try depTokenizer("\\\\foo.o",
764 \\ERROR: incomplete target '\foo.o' at position 1815 \\ERROR: incomplete target '\foo.o' at position 0
765 );816 );
766 try depTokenizer("$$foo.o",817 try depTokenizer("$$foo.o",
767 \\ERROR: incomplete target '$foo.o' at position 1818 \\ERROR: incomplete target '$foo.o' at position 0
768 );819 );
769}820}
770821
...@@ -862,7 +913,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {...@@ -862,7 +913,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
862 },913 },
863 .target_must_resolve => {914 .target_must_resolve => {
864 try buffer.appendSlice("target = {");915 try buffer.appendSlice("target = {");
865 try token.resolve(&resolve_buf);916 try token.resolve(resolve_buf.writer());
866 for (resolve_buf.items) |b| {917 for (resolve_buf.items) |b| {
867 try buffer.append(printable_char_tab[b]);918 try buffer.append(printable_char_tab[b]);
868 }919 }
...@@ -870,7 +921,9 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {...@@ -870,7 +921,9 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
870 try buffer.appendSlice("}");921 try buffer.appendSlice("}");
871 },922 },
872 else => {923 else => {
873 @panic("TODO");924 try buffer.appendSlice("ERROR: ");
925 try token.printError(buffer.outStream());
926 break;
874 },927 },
875 }928 }
876 i += 1;929 i += 1;
...@@ -1005,23 +1058,19 @@ fn printCharValues(out: anytype, bytes: []const u8) !void {...@@ -1005,23 +1058,19 @@ fn printCharValues(out: anytype, bytes: []const u8) !void {
1005 }1058 }
1006}1059}
10071060
1008fn printUnderstandableChar(buffer: *std.ArrayListSentineled(u8, 0), char: u8) !void {1061fn printUnderstandableChar(out: anytype, char: u8) !void {
1009 if (!std.ascii.isPrint(char) or char == ' ') {1062 if (!std.ascii.isPrint(char) or char == ' ') {
1010 try buffer.outStream().print("\\x{X:0>2}", .{char});1063 try out.print("\\x{X:0>2}", .{char});
1011 } else {1064 } else {
1012 try buffer.appendSlice("'");1065 try out.print("'{c}'", .{printable_char_tab[char]});
1013 try buffer.append(printable_char_tab[char]);
1014 try buffer.appendSlice("'");
1015 }1066 }
1016}1067}
10171068
1018// zig fmt: off1069// zig fmt: off
1019const printable_char_tab: []const u8 =1070const printable_char_tab: [256]u8 = (
1020 "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++1071 "................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
1021 "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++1072 "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~." ++
1022 "................................................................" ++1073 "................................................................" ++
1023 "................................................................";1074 "................................................................"
1024// zig fmt: on1075).*;
1025comptime {1076
1026 assert(printable_char_tab.len == 256);
1027}