authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-06-01 14:37:36+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-06-01 14:37:36+03:00
loga47257d9b08cf75c803c4487e90ae3e0dd7ae63a
tree83663b1773991dcc109f63a4f96ac9e8509e95bf
parent557d75d58a787ed0154e9bb2e58d26f9a5f9414e
signaturelock-open Commit is signed but in an unrecognized format.

fix std.zig rejecting literal tabs in comments

1 files changed, 75 insertions(+), 51 deletions(-)

lib/std/zig/tokenizer.zig+75-51
...@@ -11,57 +11,57 @@ pub const Token = struct {...@@ -11,57 +11,57 @@ pub const Token = struct {
11 };11 };
1212
13 pub const keywords = std.ComptimeStringMap(Id, .{13 pub const keywords = std.ComptimeStringMap(Id, .{
14 .{"align", .Keyword_align},14 .{ "align", .Keyword_align },
15 .{"allowzero", .Keyword_allowzero},15 .{ "allowzero", .Keyword_allowzero },
16 .{"and", .Keyword_and},16 .{ "and", .Keyword_and },
17 .{"anyframe", .Keyword_anyframe},17 .{ "anyframe", .Keyword_anyframe },
18 .{"asm", .Keyword_asm},18 .{ "asm", .Keyword_asm },
19 .{"async", .Keyword_async},19 .{ "async", .Keyword_async },
20 .{"await", .Keyword_await},20 .{ "await", .Keyword_await },
21 .{"break", .Keyword_break},21 .{ "break", .Keyword_break },
22 .{"callconv", .Keyword_callconv},22 .{ "callconv", .Keyword_callconv },
23 .{"catch", .Keyword_catch},23 .{ "catch", .Keyword_catch },
24 .{"comptime", .Keyword_comptime},24 .{ "comptime", .Keyword_comptime },
25 .{"const", .Keyword_const},25 .{ "const", .Keyword_const },
26 .{"continue", .Keyword_continue},26 .{ "continue", .Keyword_continue },
27 .{"defer", .Keyword_defer},27 .{ "defer", .Keyword_defer },
28 .{"else", .Keyword_else},28 .{ "else", .Keyword_else },
29 .{"enum", .Keyword_enum},29 .{ "enum", .Keyword_enum },
30 .{"errdefer", .Keyword_errdefer},30 .{ "errdefer", .Keyword_errdefer },
31 .{"error", .Keyword_error},31 .{ "error", .Keyword_error },
32 .{"export", .Keyword_export},32 .{ "export", .Keyword_export },
33 .{"extern", .Keyword_extern},33 .{ "extern", .Keyword_extern },
34 .{"false", .Keyword_false},34 .{ "false", .Keyword_false },
35 .{"fn", .Keyword_fn},35 .{ "fn", .Keyword_fn },
36 .{"for", .Keyword_for},36 .{ "for", .Keyword_for },
37 .{"if", .Keyword_if},37 .{ "if", .Keyword_if },
38 .{"inline", .Keyword_inline},38 .{ "inline", .Keyword_inline },
39 .{"noalias", .Keyword_noalias},39 .{ "noalias", .Keyword_noalias },
40 .{"noasync", .Keyword_nosuspend}, // TODO: remove this40 .{ "noasync", .Keyword_nosuspend }, // TODO: remove this
41 .{"noinline", .Keyword_noinline},41 .{ "noinline", .Keyword_noinline },
42 .{"nosuspend", .Keyword_nosuspend},42 .{ "nosuspend", .Keyword_nosuspend },
43 .{"null", .Keyword_null},43 .{ "null", .Keyword_null },
44 .{"or", .Keyword_or},44 .{ "or", .Keyword_or },
45 .{"orelse", .Keyword_orelse},45 .{ "orelse", .Keyword_orelse },
46 .{"packed", .Keyword_packed},46 .{ "packed", .Keyword_packed },
47 .{"pub", .Keyword_pub},47 .{ "pub", .Keyword_pub },
48 .{"resume", .Keyword_resume},48 .{ "resume", .Keyword_resume },
49 .{"return", .Keyword_return},49 .{ "return", .Keyword_return },
50 .{"linksection", .Keyword_linksection},50 .{ "linksection", .Keyword_linksection },
51 .{"struct", .Keyword_struct},51 .{ "struct", .Keyword_struct },
52 .{"suspend", .Keyword_suspend},52 .{ "suspend", .Keyword_suspend },
53 .{"switch", .Keyword_switch},53 .{ "switch", .Keyword_switch },
54 .{"test", .Keyword_test},54 .{ "test", .Keyword_test },
55 .{"threadlocal", .Keyword_threadlocal},55 .{ "threadlocal", .Keyword_threadlocal },
56 .{"true", .Keyword_true},56 .{ "true", .Keyword_true },
57 .{"try", .Keyword_try},57 .{ "try", .Keyword_try },
58 .{"undefined", .Keyword_undefined},58 .{ "undefined", .Keyword_undefined },
59 .{"union", .Keyword_union},59 .{ "union", .Keyword_union },
60 .{"unreachable", .Keyword_unreachable},60 .{ "unreachable", .Keyword_unreachable },
61 .{"usingnamespace", .Keyword_usingnamespace},61 .{ "usingnamespace", .Keyword_usingnamespace },
62 .{"var", .Keyword_var},62 .{ "var", .Keyword_var },
63 .{"volatile", .Keyword_volatile},63 .{ "volatile", .Keyword_volatile },
64 .{"while", .Keyword_while},64 .{ "while", .Keyword_while },
65 });65 });
6666
67 pub fn getKeyword(bytes: []const u8) ?Id {67 pub fn getKeyword(bytes: []const u8) ?Id {
...@@ -1014,6 +1014,7 @@ pub const Tokenizer = struct {...@@ -1014,6 +1014,7 @@ pub const Tokenizer = struct {
1014 state = .container_doc_comment;1014 state = .container_doc_comment;
1015 },1015 },
1016 '\n' => break,1016 '\n' => break,
1017 '\t' => state = .line_comment,
1017 else => {1018 else => {
1018 state = .line_comment;1019 state = .line_comment;
1019 self.checkLiteralCharacter();1020 self.checkLiteralCharacter();
...@@ -1027,6 +1028,10 @@ pub const Tokenizer = struct {...@@ -1027,6 +1028,10 @@ pub const Tokenizer = struct {
1027 result.id = .DocComment;1028 result.id = .DocComment;
1028 break;1029 break;
1029 },1030 },
1031 '\t' => {
1032 state = .doc_comment;
1033 result.id = .DocComment;
1034 },
1030 else => {1035 else => {
1031 state = .doc_comment;1036 state = .doc_comment;
1032 result.id = .DocComment;1037 result.id = .DocComment;
...@@ -1035,6 +1040,7 @@ pub const Tokenizer = struct {...@@ -1035,6 +1040,7 @@ pub const Tokenizer = struct {
1035 },1040 },
1036 .line_comment, .doc_comment, .container_doc_comment => switch (c) {1041 .line_comment, .doc_comment, .container_doc_comment => switch (c) {
1037 '\n' => break,1042 '\n' => break,
1043 '\t' => {},
1038 else => self.checkLiteralCharacter(),1044 else => self.checkLiteralCharacter(),
1039 },1045 },
1040 .zero => switch (c) {1046 .zero => switch (c) {
...@@ -1677,6 +1683,24 @@ test "tokenizer - multiline string literal with literal tab" {...@@ -1677,6 +1683,24 @@ test "tokenizer - multiline string literal with literal tab" {
1677 });1683 });
1678}1684}
16791685
1686test "tokenizer - comments with literal tab" {
1687 testTokenize(
1688 \\//foo bar
1689 \\//!foo bar
1690 \\///foo bar
1691 \\// foo
1692 \\/// foo
1693 \\/// /foo
1694 , &[_]Token.Id{
1695 .LineComment,
1696 .ContainerDocComment,
1697 .DocComment,
1698 .LineComment,
1699 .DocComment,
1700 .DocComment,
1701 });
1702}
1703
1680test "tokenizer - pipe and then invalid" {1704test "tokenizer - pipe and then invalid" {
1681 testTokenize("||=", &[_]Token.Id{1705 testTokenize("||=", &[_]Token.Id{
1682 .PipePipe,1706 .PipePipe,