authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-06-01 17:24:30+03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-06-01 17:24:30+03:00
log078e4afdaf496d83799b6770d5e11ecf099b1e42
treecc7ea0073cc06023ed2a1bf7f72b3ac0ae147ccc
parenteb687810cfdb057b661148a759280d83e6135393
parenta47257d9b08cf75c803c4487e90ae3e0dd7ae63a
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5500 from Vexu/fix

Fix std.zig rejecting literal tabs in comments

1 files changed, 75 insertions(+), 51 deletions(-)

lib/std/zig/tokenizer.zig+75-51
......@@ -11,57 +11,57 @@ pub const Token = struct {
1111 };
1212
1313 pub const keywords = std.ComptimeStringMap(Id, .{
14 .{"align", .Keyword_align},
15 .{"allowzero", .Keyword_allowzero},
16 .{"and", .Keyword_and},
17 .{"anyframe", .Keyword_anyframe},
18 .{"asm", .Keyword_asm},
19 .{"async", .Keyword_async},
20 .{"await", .Keyword_await},
21 .{"break", .Keyword_break},
22 .{"callconv", .Keyword_callconv},
23 .{"catch", .Keyword_catch},
24 .{"comptime", .Keyword_comptime},
25 .{"const", .Keyword_const},
26 .{"continue", .Keyword_continue},
27 .{"defer", .Keyword_defer},
28 .{"else", .Keyword_else},
29 .{"enum", .Keyword_enum},
30 .{"errdefer", .Keyword_errdefer},
31 .{"error", .Keyword_error},
32 .{"export", .Keyword_export},
33 .{"extern", .Keyword_extern},
34 .{"false", .Keyword_false},
35 .{"fn", .Keyword_fn},
36 .{"for", .Keyword_for},
37 .{"if", .Keyword_if},
38 .{"inline", .Keyword_inline},
39 .{"noalias", .Keyword_noalias},
40 .{"noasync", .Keyword_nosuspend}, // TODO: remove this
41 .{"noinline", .Keyword_noinline},
42 .{"nosuspend", .Keyword_nosuspend},
43 .{"null", .Keyword_null},
44 .{"or", .Keyword_or},
45 .{"orelse", .Keyword_orelse},
46 .{"packed", .Keyword_packed},
47 .{"pub", .Keyword_pub},
48 .{"resume", .Keyword_resume},
49 .{"return", .Keyword_return},
50 .{"linksection", .Keyword_linksection},
51 .{"struct", .Keyword_struct},
52 .{"suspend", .Keyword_suspend},
53 .{"switch", .Keyword_switch},
54 .{"test", .Keyword_test},
55 .{"threadlocal", .Keyword_threadlocal},
56 .{"true", .Keyword_true},
57 .{"try", .Keyword_try},
58 .{"undefined", .Keyword_undefined},
59 .{"union", .Keyword_union},
60 .{"unreachable", .Keyword_unreachable},
61 .{"usingnamespace", .Keyword_usingnamespace},
62 .{"var", .Keyword_var},
63 .{"volatile", .Keyword_volatile},
64 .{"while", .Keyword_while},
14 .{ "align", .Keyword_align },
15 .{ "allowzero", .Keyword_allowzero },
16 .{ "and", .Keyword_and },
17 .{ "anyframe", .Keyword_anyframe },
18 .{ "asm", .Keyword_asm },
19 .{ "async", .Keyword_async },
20 .{ "await", .Keyword_await },
21 .{ "break", .Keyword_break },
22 .{ "callconv", .Keyword_callconv },
23 .{ "catch", .Keyword_catch },
24 .{ "comptime", .Keyword_comptime },
25 .{ "const", .Keyword_const },
26 .{ "continue", .Keyword_continue },
27 .{ "defer", .Keyword_defer },
28 .{ "else", .Keyword_else },
29 .{ "enum", .Keyword_enum },
30 .{ "errdefer", .Keyword_errdefer },
31 .{ "error", .Keyword_error },
32 .{ "export", .Keyword_export },
33 .{ "extern", .Keyword_extern },
34 .{ "false", .Keyword_false },
35 .{ "fn", .Keyword_fn },
36 .{ "for", .Keyword_for },
37 .{ "if", .Keyword_if },
38 .{ "inline", .Keyword_inline },
39 .{ "noalias", .Keyword_noalias },
40 .{ "noasync", .Keyword_nosuspend }, // TODO: remove this
41 .{ "noinline", .Keyword_noinline },
42 .{ "nosuspend", .Keyword_nosuspend },
43 .{ "null", .Keyword_null },
44 .{ "or", .Keyword_or },
45 .{ "orelse", .Keyword_orelse },
46 .{ "packed", .Keyword_packed },
47 .{ "pub", .Keyword_pub },
48 .{ "resume", .Keyword_resume },
49 .{ "return", .Keyword_return },
50 .{ "linksection", .Keyword_linksection },
51 .{ "struct", .Keyword_struct },
52 .{ "suspend", .Keyword_suspend },
53 .{ "switch", .Keyword_switch },
54 .{ "test", .Keyword_test },
55 .{ "threadlocal", .Keyword_threadlocal },
56 .{ "true", .Keyword_true },
57 .{ "try", .Keyword_try },
58 .{ "undefined", .Keyword_undefined },
59 .{ "union", .Keyword_union },
60 .{ "unreachable", .Keyword_unreachable },
61 .{ "usingnamespace", .Keyword_usingnamespace },
62 .{ "var", .Keyword_var },
63 .{ "volatile", .Keyword_volatile },
64 .{ "while", .Keyword_while },
6565 });
6666
6767 pub fn getKeyword(bytes: []const u8) ?Id {
......@@ -1014,6 +1014,7 @@ pub const Tokenizer = struct {
10141014 state = .container_doc_comment;
10151015 },
10161016 '\n' => break,
1017 '\t' => state = .line_comment,
10171018 else => {
10181019 state = .line_comment;
10191020 self.checkLiteralCharacter();
......@@ -1027,6 +1028,10 @@ pub const Tokenizer = struct {
10271028 result.id = .DocComment;
10281029 break;
10291030 },
1031 '\t' => {
1032 state = .doc_comment;
1033 result.id = .DocComment;
1034 },
10301035 else => {
10311036 state = .doc_comment;
10321037 result.id = .DocComment;
......@@ -1035,6 +1040,7 @@ pub const Tokenizer = struct {
10351040 },
10361041 .line_comment, .doc_comment, .container_doc_comment => switch (c) {
10371042 '\n' => break,
1043 '\t' => {},
10381044 else => self.checkLiteralCharacter(),
10391045 },
10401046 .zero => switch (c) {
......@@ -1677,6 +1683,24 @@ test "tokenizer - multiline string literal with literal tab" {
16771683 });
16781684}
16791685
1686test "tokenizer - comments with literal tab" {
1687 testTokenize(
1688 \\//foo bar
1689 \\//!foo bar
1690 \\///foo bar
1691 \\// foo
1692 \\/// foo
1693 \\/// /foo
1694 , &[_]Token.Id{
1695 .LineComment,
1696 .ContainerDocComment,
1697 .DocComment,
1698 .LineComment,
1699 .DocComment,
1700 .DocComment,
1701 });
1702}
1703
16801704test "tokenizer - pipe and then invalid" {
16811705 testTokenize("||=", &[_]Token.Id{
16821706 .PipePipe,