authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-07-02 12:33:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-07-02 13:28:31-07:00
logc5c23db6278332044c0606d78419000b68185e0c
treee0534dd3eaf43d5dd776eb1120ca364810546283
parent7a2e0d98109d39f06400dfbc03c12695557100c6

tokenizer: clean up invalid token error

It now displays the byte with proper printability handling. This makes the relevant compile error test case no longer a regression in quality from stage1 to stage2.

7 files changed, 56 insertions(+), 46 deletions(-)

lib/std/zig/tokenizer.zig+2-1
......@@ -551,8 +551,9 @@ pub const Tokenizer = struct {
551551 },
552552 else => {
553553 result.tag = .invalid;
554 result.loc.end = self.index;
554555 self.index += 1;
555 break;
556 return result;
556557 },
557558 },
558559
src/Module.zig+3-2
......@@ -2480,11 +2480,12 @@ pub fn astGenFile(mod: *Module, file: *Scope.File) !void {
24802480 };
24812481 if (token_tags[parse_err.token] == .invalid) {
24822482 const bad_off = @intCast(u32, file.tree.tokenSlice(parse_err.token).len);
2483 const byte_abs = token_starts[parse_err.token] + bad_off;
24832484 try mod.errNoteNonLazy(.{
24842485 .file_scope = file,
24852486 .parent_decl_node = 0,
2486 .lazy = .{ .byte_abs = token_starts[parse_err.token] + bad_off },
2487 }, err_msg, "invalid byte here", .{});
2487 .lazy = .{ .byte_abs = byte_abs },
2488 }, err_msg, "invalid byte: '{'}'", .{ std.zig.fmtEscapes(source[byte_abs..][0..1]) });
24882489 }
24892490
24902491 {
src/main.zig+15-9
......@@ -233,7 +233,7 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v
233233 } else if (mem.eql(u8, cmd, "build")) {
234234 return cmdBuild(gpa, arena, cmd_args);
235235 } else if (mem.eql(u8, cmd, "fmt")) {
236 return cmdFmt(gpa, cmd_args);
236 return cmdFmt(gpa, arena, cmd_args);
237237 } else if (mem.eql(u8, cmd, "libc")) {
238238 return cmdLibC(gpa, cmd_args);
239239 } else if (mem.eql(u8, cmd, "init-exe")) {
......@@ -3039,12 +3039,13 @@ const Fmt = struct {
30393039 check_ast: bool,
30403040 color: Color,
30413041 gpa: *Allocator,
3042 arena: *Allocator,
30423043 out_buffer: std.ArrayList(u8),
30433044
30443045 const SeenMap = std.AutoHashMap(fs.File.INode, void);
30453046};
30463047
3047pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
3048pub fn cmdFmt(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !void {
30483049 var color: Color = .auto;
30493050 var stdin_flag: bool = false;
30503051 var check_flag: bool = false;
......@@ -3102,7 +3103,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
31023103 defer tree.deinit(gpa);
31033104
31043105 for (tree.errors) |parse_error| {
3105 try printErrMsgToStdErr(gpa, parse_error, tree, "<stdin>", color);
3106 try printErrMsgToStdErr(gpa, arena, parse_error, tree, "<stdin>", color);
31063107 }
31073108 var has_ast_error = false;
31083109 if (check_ast_flag) {
......@@ -3170,6 +3171,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void {
31703171
31713172 var fmt = Fmt{
31723173 .gpa = gpa,
3174 .arena = arena,
31733175 .seen = Fmt.SeenMap.init(gpa),
31743176 .any_error = false,
31753177 .check_ast = check_ast_flag,
......@@ -3293,7 +3295,7 @@ fn fmtPathFile(
32933295 defer tree.deinit(fmt.gpa);
32943296
32953297 for (tree.errors) |parse_error| {
3296 try printErrMsgToStdErr(fmt.gpa, parse_error, tree, file_path, fmt.color);
3298 try printErrMsgToStdErr(fmt.gpa, fmt.arena, parse_error, tree, file_path, fmt.color);
32973299 }
32983300 if (tree.errors.len != 0) {
32993301 fmt.any_error = true;
......@@ -3374,6 +3376,7 @@ fn fmtPathFile(
33743376
33753377fn printErrMsgToStdErr(
33763378 gpa: *mem.Allocator,
3379 arena: *mem.Allocator,
33773380 parse_error: ast.Error,
33783381 tree: ast.Tree,
33793382 path: []const u8,
......@@ -3395,11 +3398,14 @@ fn printErrMsgToStdErr(
33953398
33963399 if (token_tags[parse_error.token] == .invalid) {
33973400 const bad_off = @intCast(u32, tree.tokenSlice(parse_error.token).len);
3401 const byte_offset = @intCast(u32, start_loc.line_start) + bad_off;
33983402 notes_buffer[notes_len] = .{
33993403 .src = .{
34003404 .src_path = path,
3401 .msg = "invalid byte here",
3402 .byte_offset = @intCast(u32, start_loc.line_start) + bad_off,
3405 .msg = try std.fmt.allocPrint(arena, "invalid byte: '{'}'", .{
3406 std.zig.fmtEscapes(tree.source[byte_offset..][0..1]),
3407 }),
3408 .byte_offset = byte_offset,
34033409 .line = @intCast(u32, start_loc.line),
34043410 .column = @intCast(u32, start_loc.column) + bad_off,
34053411 .source_line = source_line,
......@@ -3943,7 +3949,7 @@ pub fn cmdAstCheck(
39433949 defer file.tree.deinit(gpa);
39443950
39453951 for (file.tree.errors) |parse_error| {
3946 try printErrMsgToStdErr(gpa, parse_error, file.tree, file.sub_file_path, color);
3952 try printErrMsgToStdErr(gpa, arena, parse_error, file.tree, file.sub_file_path, color);
39473953 }
39483954 if (file.tree.errors.len != 0) {
39493955 process.exit(1);
......@@ -4069,7 +4075,7 @@ pub fn cmdChangelist(
40694075 defer file.tree.deinit(gpa);
40704076
40714077 for (file.tree.errors) |parse_error| {
4072 try printErrMsgToStdErr(gpa, parse_error, file.tree, old_source_file, .auto);
4078 try printErrMsgToStdErr(gpa, arena, parse_error, file.tree, old_source_file, .auto);
40734079 }
40744080 if (file.tree.errors.len != 0) {
40754081 process.exit(1);
......@@ -4108,7 +4114,7 @@ pub fn cmdChangelist(
41084114 defer new_tree.deinit(gpa);
41094115
41104116 for (new_tree.errors) |parse_error| {
4111 try printErrMsgToStdErr(gpa, parse_error, new_tree, new_source_file, .auto);
4117 try printErrMsgToStdErr(gpa, arena, parse_error, new_tree, new_source_file, .auto);
41124118 }
41134119 if (new_tree.errors.len != 0) {
41144120 process.exit(1);
src/stage1/analyze.cpp+3-3
......@@ -3915,7 +3915,7 @@ static void add_top_level_decl(CodeGen *g, ScopeDecls *decls_scope, Tld *tld) {
39153915 }
39163916 }
39173917 ErrorMsg *msg = add_node_error(g, tld->source_node, buf_sprintf("redefinition of '%s'", buf_ptr(tld->name)));
3918 add_error_note(g, msg, other_tld->source_node, buf_sprintf("previous definition is here"));
3918 add_error_note(g, msg, other_tld->source_node, buf_sprintf("previous definition here"));
39193919 return;
39203920 }
39213921
......@@ -4176,7 +4176,7 @@ ZigVar *add_variable(CodeGen *g, AstNode *source_node, Scope *parent_scope, Buf
41764176 if (existing_var->var_type == nullptr || !type_is_invalid(existing_var->var_type)) {
41774177 ErrorMsg *msg = add_node_error(g, source_node,
41784178 buf_sprintf("redeclaration of variable '%s'", buf_ptr(name)));
4179 add_error_note(g, msg, existing_var->decl_node, buf_sprintf("previous declaration is here"));
4179 add_error_note(g, msg, existing_var->decl_node, buf_sprintf("previous declaration here"));
41804180 }
41814181 variable_entry->var_type = g->builtin_types.entry_invalid;
41824182 } else {
......@@ -4205,7 +4205,7 @@ ZigVar *add_variable(CodeGen *g, AstNode *source_node, Scope *parent_scope, Buf
42054205 if (want_err_msg) {
42064206 ErrorMsg *msg = add_node_error(g, source_node,
42074207 buf_sprintf("redefinition of '%s'", buf_ptr(name)));
4208 add_error_note(g, msg, tld->source_node, buf_sprintf("previous definition is here"));
4208 add_error_note(g, msg, tld->source_node, buf_sprintf("previous definition here"));
42094209 }
42104210 variable_entry->var_type = g->builtin_types.entry_invalid;
42114211 }
test/cases.zig+5-5
......@@ -1039,8 +1039,8 @@ pub fn addCases(ctx: *TestContext) !void {
10391039 \\ var i: u32 = 10;
10401040 \\}
10411041 , &[_][]const u8{
1042 ":3:9: error: redeclaration of 'i'",
1043 ":2:9: note: previously declared here",
1042 ":3:9: error: redeclaration of local variable 'i'",
1043 ":2:9: note: previous declaration here",
10441044 });
10451045 case.addError(
10461046 \\var testing: i64 = 10;
......@@ -1061,8 +1061,8 @@ pub fn addCases(ctx: *TestContext) !void {
10611061 \\ };
10621062 \\}
10631063 , &[_][]const u8{
1064 ":5:19: error: redeclaration of 'c'",
1065 ":4:19: note: previously declared here",
1064 ":5:19: error: redeclaration of local constant 'c'",
1065 ":4:19: note: previous declaration here",
10661066 });
10671067 }
10681068
......@@ -1214,7 +1214,7 @@ pub fn addCases(ctx: *TestContext) !void {
12141214 \\}
12151215 , &[_][]const u8{
12161216 ":2:11: error: redefinition of label 'blk'",
1217 ":2:5: note: previous definition is here",
1217 ":2:5: note: previous definition here",
12181218 });
12191219 }
12201220
test/compile_errors.zig+27-26
......@@ -1507,7 +1507,7 @@ pub fn addCases(ctx: *TestContext) !void {
15071507 \\}
15081508 , &[_][]const u8{
15091509 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1510 "tmp.zig:2:28: note: invalid byte here",
1510 "tmp.zig:2:28: note: invalid byte: 'a'",
15111511 });
15121512
15131513 ctx.objErrStage1("invalid exponent in float literal - 2",
......@@ -1517,7 +1517,7 @@ pub fn addCases(ctx: *TestContext) !void {
15171517 \\}
15181518 , &[_][]const u8{
15191519 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1520 "tmp.zig:2:29: note: invalid byte here",
1520 "tmp.zig:2:29: note: invalid byte: 'F'",
15211521 });
15221522
15231523 ctx.objErrStage1("invalid underscore placement in float literal - 1",
......@@ -1527,7 +1527,7 @@ pub fn addCases(ctx: *TestContext) !void {
15271527 \\}
15281528 , &[_][]const u8{
15291529 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1530 "tmp.zig:2:23: note: invalid byte here",
1530 "tmp.zig:2:23: note: invalid byte: '_'",
15311531 });
15321532
15331533 ctx.objErrStage1("invalid underscore placement in float literal - 2",
......@@ -1537,7 +1537,7 @@ pub fn addCases(ctx: *TestContext) !void {
15371537 \\}
15381538 , &[_][]const u8{
15391539 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1540 "tmp.zig:2:23: note: invalid byte here",
1540 "tmp.zig:2:23: note: invalid byte: '.'",
15411541 });
15421542
15431543 ctx.objErrStage1("invalid underscore placement in float literal - 3",
......@@ -1547,7 +1547,7 @@ pub fn addCases(ctx: *TestContext) !void {
15471547 \\}
15481548 , &[_][]const u8{
15491549 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1550 "tmp.zig:2:25: note: invalid byte here",
1550 "tmp.zig:2:25: note: invalid byte: ';'",
15511551 });
15521552
15531553 ctx.objErrStage1("invalid underscore placement in float literal - 4",
......@@ -1557,7 +1557,7 @@ pub fn addCases(ctx: *TestContext) !void {
15571557 \\}
15581558 , &[_][]const u8{
15591559 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1560 "tmp.zig:2:25: note: invalid byte here",
1560 "tmp.zig:2:25: note: invalid byte: '_'",
15611561 });
15621562
15631563 ctx.objErrStage1("invalid underscore placement in float literal - 5",
......@@ -1567,7 +1567,7 @@ pub fn addCases(ctx: *TestContext) !void {
15671567 \\}
15681568 , &[_][]const u8{
15691569 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1570 "tmp.zig:2:26: note: invalid byte here",
1570 "tmp.zig:2:26: note: invalid byte: '_'",
15711571 });
15721572
15731573 ctx.objErrStage1("invalid underscore placement in float literal - 6",
......@@ -1577,7 +1577,7 @@ pub fn addCases(ctx: *TestContext) !void {
15771577 \\}
15781578 , &[_][]const u8{
15791579 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1580 "tmp.zig:2:26: note: invalid byte here",
1580 "tmp.zig:2:26: note: invalid byte: '_'",
15811581 });
15821582
15831583 ctx.objErrStage1("invalid underscore placement in float literal - 7",
......@@ -1587,7 +1587,7 @@ pub fn addCases(ctx: *TestContext) !void {
15871587 \\}
15881588 , &[_][]const u8{
15891589 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1590 "tmp.zig:2:28: note: invalid byte here",
1590 "tmp.zig:2:28: note: invalid byte: ';'",
15911591 });
15921592
15931593 ctx.objErrStage1("invalid underscore placement in float literal - 9",
......@@ -1597,7 +1597,7 @@ pub fn addCases(ctx: *TestContext) !void {
15971597 \\}
15981598 , &[_][]const u8{
15991599 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1600 "tmp.zig:2:23: note: invalid byte here",
1600 "tmp.zig:2:23: note: invalid byte: '_'",
16011601 });
16021602
16031603 ctx.objErrStage1("invalid underscore placement in float literal - 10",
......@@ -1607,7 +1607,7 @@ pub fn addCases(ctx: *TestContext) !void {
16071607 \\}
16081608 , &[_][]const u8{
16091609 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1610 "tmp.zig:2:25: note: invalid byte here",
1610 "tmp.zig:2:25: note: invalid byte: '_'",
16111611 });
16121612
16131613 ctx.objErrStage1("invalid underscore placement in float literal - 11",
......@@ -1617,7 +1617,7 @@ pub fn addCases(ctx: *TestContext) !void {
16171617 \\}
16181618 , &[_][]const u8{
16191619 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1620 "tmp.zig:2:28: note: invalid byte here",
1620 "tmp.zig:2:28: note: invalid byte: '_'",
16211621 });
16221622
16231623 ctx.objErrStage1("invalid underscore placement in float literal - 12",
......@@ -1627,7 +1627,7 @@ pub fn addCases(ctx: *TestContext) !void {
16271627 \\}
16281628 , &[_][]const u8{
16291629 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1630 "tmp.zig:2:23: note: invalid byte here",
1630 "tmp.zig:2:23: note: invalid byte: 'x'",
16311631 });
16321632
16331633 ctx.objErrStage1("invalid underscore placement in float literal - 13",
......@@ -1637,7 +1637,7 @@ pub fn addCases(ctx: *TestContext) !void {
16371637 \\}
16381638 , &[_][]const u8{
16391639 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1640 "tmp.zig:2:23: note: invalid byte here",
1640 "tmp.zig:2:23: note: invalid byte: '_'",
16411641 });
16421642
16431643 ctx.objErrStage1("invalid underscore placement in float literal - 14",
......@@ -1647,7 +1647,7 @@ pub fn addCases(ctx: *TestContext) !void {
16471647 \\}
16481648 , &[_][]const u8{
16491649 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1650 "tmp.zig:2:27: note: invalid byte here",
1650 "tmp.zig:2:27: note: invalid byte: 'p'",
16511651 });
16521652
16531653 ctx.objErrStage1("invalid underscore placement in int literal - 1",
......@@ -1657,7 +1657,7 @@ pub fn addCases(ctx: *TestContext) !void {
16571657 \\}
16581658 , &[_][]const u8{
16591659 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1660 "tmp.zig:2:26: note: invalid byte here",
1660 "tmp.zig:2:26: note: invalid byte: ';'",
16611661 });
16621662
16631663 ctx.objErrStage1("invalid underscore placement in int literal - 2",
......@@ -1667,7 +1667,7 @@ pub fn addCases(ctx: *TestContext) !void {
16671667 \\}
16681668 , &[_][]const u8{
16691669 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1670 "tmp.zig:2:28: note: invalid byte here",
1670 "tmp.zig:2:28: note: invalid byte: ';'",
16711671 });
16721672
16731673 ctx.objErrStage1("invalid underscore placement in int literal - 3",
......@@ -1677,7 +1677,7 @@ pub fn addCases(ctx: *TestContext) !void {
16771677 \\}
16781678 , &[_][]const u8{
16791679 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1680 "tmp.zig:2:28: note: invalid byte here",
1680 "tmp.zig:2:28: note: invalid byte: ';'",
16811681 });
16821682
16831683 ctx.objErrStage1("invalid underscore placement in int literal - 4",
......@@ -1687,7 +1687,7 @@ pub fn addCases(ctx: *TestContext) !void {
16871687 \\}
16881688 , &[_][]const u8{
16891689 "tmp.zig:2:21: error: expected expression, found 'invalid'",
1690 "tmp.zig:2:28: note: invalid byte here",
1690 "tmp.zig:2:28: note: invalid byte: ';'",
16911691 });
16921692
16931693 ctx.objErrStage1("comptime struct field, no init value",
......@@ -4932,10 +4932,10 @@ pub fn addCases(ctx: *TestContext) !void {
49324932 });
49334933
49344934 ctx.objErrStage1("wrong number of arguments",
4935 \\export fn d() void {
4936 \\ e(1);
4935 \\export fn a() void {
4936 \\ c(1);
49374937 \\}
4938 \\fn b(a: i32, b: i32, c: i32) void { _ = a; _ = b; _ = c; }
4938 \\fn c(d: i32, e: i32, f: i32) void { _ = d; _ = e; _ = f; }
49394939 , &[_][]const u8{
49404940 "tmp.zig:2:6: error: expected 3 argument(s), found 1",
49414941 });
......@@ -5669,7 +5669,7 @@ pub fn addCases(ctx: *TestContext) !void {
56695669 \\b";
56705670 , &[_][]const u8{
56715671 "tmp.zig:1:13: error: expected expression, found 'invalid'",
5672 "tmp.zig:1:15: note: invalid byte here",
5672 "tmp.zig:1:15: note: invalid byte: '\\n'",
56735673 });
56745674
56755675 ctx.objErrStage1("invalid comparison for function pointers",
......@@ -7569,7 +7569,7 @@ pub fn addCases(ctx: *TestContext) !void {
75697569 \\}
75707570 , &[_][]const u8{
75717571 "tmp.zig:2:15: error: expected expression, found 'invalid'",
7572 "tmp.zig:2:18: note: invalid byte here",
7572 "tmp.zig:2:18: note: invalid byte: '1'",
75737573 });
75747574
75757575 ctx.objErrStage1("invalid empty unicode escape",
......@@ -7584,7 +7584,8 @@ pub fn addCases(ctx: *TestContext) !void {
75847584 "fn foo() bool {\r\n" ++
75857585 " return true;\r\n" ++
75867586 "}\r\n", &[_][]const u8{
7587 "tmp.zig:1:1: error: invalid character: '\\xff'",
7587 "tmp.zig:1:1: error: expected test, comptime, var decl, or container field, found 'invalid'",
7588 "tmp.zig:1:1: note: invalid byte: '\\xff'",
75887589 });
75897590
75907591 ctx.objErrStage1("non-printable invalid character with escape alternative", "fn foo() bool {\n" ++
......@@ -8769,7 +8770,7 @@ pub fn addCases(ctx: *TestContext) !void {
87698770 \\}
87708771 , &[_][]const u8{
87718772 // Ideally this would be column 30 but it's not very important.
8772 "tmp.zig:2:28: error: `.*` cannot be followed by `*`. Are you missing a space?",
8773 "tmp.zig:2:28: error: '.*' cannot be followed by '*'. Are you missing a space?",
87738774 });
87748775
87758776 ctx.objErrStage1("Issue #9165: windows tcp server compilation error",
test/stage2/cbe.zig+1
......@@ -591,6 +591,7 @@ pub fn addCases(ctx: *TestContext) !void {
591591 , &.{
592592 ":3:5: error: enum fields cannot be marked comptime",
593593 ":8:8: error: enum fields do not have types",
594 ":6:12: note: consider 'union(enum)' here to make it a tagged union",
594595 });
595596
596597 // @enumToInt, @intToEnum, enum literal coercion, field access syntax, comparison, switch