authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2020-05-26 21:36:58-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2020-05-26 21:36:58-07:00
loga9dd79d29359760f46d4a28c4dcf65b720b7b755
tree9d248adb9c66d8e6772255029ee3f949b46274fa
parent0865e5d360500e101fbf5e23b842bb6e26435a38

Use std.ComptimeStringMap in zig's tokenizer

Should have no performance change, the implementation was just made generic/re-usable

1 files changed, 54 insertions(+), 107 deletions(-)

lib/std/zig/tokenizer.zig+54-107
......@@ -10,115 +10,62 @@ pub const Token = struct {
1010 end: usize,
1111 };
1212
13 pub const Keyword = struct {
14 bytes: []const u8,
15 id: Id,
16
17 fn init(bytes: []const u8, id: Id) Keyword {
18 return .{
19 .bytes = bytes,
20 .id = id,
21 };
22 }
23 };
24
25 pub const keywords = [_]Keyword{
26 Keyword.init("align", .Keyword_align),
27 Keyword.init("allowzero", .Keyword_allowzero),
28 Keyword.init("and", .Keyword_and),
29 Keyword.init("anyframe", .Keyword_anyframe),
30 Keyword.init("asm", .Keyword_asm),
31 Keyword.init("async", .Keyword_async),
32 Keyword.init("await", .Keyword_await),
33 Keyword.init("break", .Keyword_break),
34 Keyword.init("callconv", .Keyword_callconv),
35 Keyword.init("catch", .Keyword_catch),
36 Keyword.init("comptime", .Keyword_comptime),
37 Keyword.init("const", .Keyword_const),
38 Keyword.init("continue", .Keyword_continue),
39 Keyword.init("defer", .Keyword_defer),
40 Keyword.init("else", .Keyword_else),
41 Keyword.init("enum", .Keyword_enum),
42 Keyword.init("errdefer", .Keyword_errdefer),
43 Keyword.init("error", .Keyword_error),
44 Keyword.init("export", .Keyword_export),
45 Keyword.init("extern", .Keyword_extern),
46 Keyword.init("false", .Keyword_false),
47 Keyword.init("fn", .Keyword_fn),
48 Keyword.init("for", .Keyword_for),
49 Keyword.init("if", .Keyword_if),
50 Keyword.init("inline", .Keyword_inline),
51 Keyword.init("noalias", .Keyword_noalias),
52 Keyword.init("noasync", .Keyword_nosuspend), // TODO: remove this
53 Keyword.init("noinline", .Keyword_noinline),
54 Keyword.init("nosuspend", .Keyword_nosuspend),
55 Keyword.init("null", .Keyword_null),
56 Keyword.init("or", .Keyword_or),
57 Keyword.init("orelse", .Keyword_orelse),
58 Keyword.init("packed", .Keyword_packed),
59 Keyword.init("pub", .Keyword_pub),
60 Keyword.init("resume", .Keyword_resume),
61 Keyword.init("return", .Keyword_return),
62 Keyword.init("linksection", .Keyword_linksection),
63 Keyword.init("struct", .Keyword_struct),
64 Keyword.init("suspend", .Keyword_suspend),
65 Keyword.init("switch", .Keyword_switch),
66 Keyword.init("test", .Keyword_test),
67 Keyword.init("threadlocal", .Keyword_threadlocal),
68 Keyword.init("true", .Keyword_true),
69 Keyword.init("try", .Keyword_try),
70 Keyword.init("undefined", .Keyword_undefined),
71 Keyword.init("union", .Keyword_union),
72 Keyword.init("unreachable", .Keyword_unreachable),
73 Keyword.init("usingnamespace", .Keyword_usingnamespace),
74 Keyword.init("var", .Keyword_var),
75 Keyword.init("volatile", .Keyword_volatile),
76 Keyword.init("while", .Keyword_while),
77 };
13 pub const keywords = std.ComptimeStringMap(Id, .{
14 .{"align", .Keyword_align},
15 .{"allowzero", .Keyword_allowzero},
16 .{"and", .Keyword_and},
17 .{"anyframe", .Keyword_anyframe},
18 .{"asm", .Keyword_asm},
19 .{"async", .Keyword_async},
20 .{"await", .Keyword_await},
21 .{"break", .Keyword_break},
22 .{"callconv", .Keyword_callconv},
23 .{"catch", .Keyword_catch},
24 .{"comptime", .Keyword_comptime},
25 .{"const", .Keyword_const},
26 .{"continue", .Keyword_continue},
27 .{"defer", .Keyword_defer},
28 .{"else", .Keyword_else},
29 .{"enum", .Keyword_enum},
30 .{"errdefer", .Keyword_errdefer},
31 .{"error", .Keyword_error},
32 .{"export", .Keyword_export},
33 .{"extern", .Keyword_extern},
34 .{"false", .Keyword_false},
35 .{"fn", .Keyword_fn},
36 .{"for", .Keyword_for},
37 .{"if", .Keyword_if},
38 .{"inline", .Keyword_inline},
39 .{"noalias", .Keyword_noalias},
40 .{"noasync", .Keyword_nosuspend}, // TODO: remove this
41 .{"noinline", .Keyword_noinline},
42 .{"nosuspend", .Keyword_nosuspend},
43 .{"null", .Keyword_null},
44 .{"or", .Keyword_or},
45 .{"orelse", .Keyword_orelse},
46 .{"packed", .Keyword_packed},
47 .{"pub", .Keyword_pub},
48 .{"resume", .Keyword_resume},
49 .{"return", .Keyword_return},
50 .{"linksection", .Keyword_linksection},
51 .{"struct", .Keyword_struct},
52 .{"suspend", .Keyword_suspend},
53 .{"switch", .Keyword_switch},
54 .{"test", .Keyword_test},
55 .{"threadlocal", .Keyword_threadlocal},
56 .{"true", .Keyword_true},
57 .{"try", .Keyword_try},
58 .{"undefined", .Keyword_undefined},
59 .{"union", .Keyword_union},
60 .{"unreachable", .Keyword_unreachable},
61 .{"usingnamespace", .Keyword_usingnamespace},
62 .{"var", .Keyword_var},
63 .{"volatile", .Keyword_volatile},
64 .{"while", .Keyword_while},
65 });
7866
7967 pub fn getKeyword(bytes: []const u8) ?Id {
80 const precomputed = comptime blk: {
81 @setEvalBranchQuota(2000);
82 var sorted_keywords = keywords;
83 const lenAsc = (struct {
84 fn lenAsc(a: Keyword, b: Keyword) bool {
85 return a.bytes.len < b.bytes.len;
86 }
87 }).lenAsc;
88 std.sort.sort(Keyword, &sorted_keywords, lenAsc);
89 const min_len = sorted_keywords[0].bytes.len;
90 const max_len = sorted_keywords[sorted_keywords.len - 1].bytes.len;
91 var len_indexes: [max_len + 1]usize = undefined;
92 var len: usize = 0;
93 var kw_i: usize = 0;
94 while (len <= max_len) : (len += 1) {
95 // find the first keyword len == len
96 while (len > sorted_keywords[kw_i].bytes.len) {
97 kw_i += 1;
98 }
99 len_indexes[len] = kw_i;
100 }
101 break :blk .{
102 .min_len = min_len,
103 .max_len = max_len,
104 .sorted_keywords = sorted_keywords,
105 .len_indexes = len_indexes,
106 };
107 };
108 if (bytes.len < precomputed.min_len or bytes.len > precomputed.max_len)
109 return null;
110
111 var i = precomputed.len_indexes[bytes.len];
112 while (true) {
113 const kw = precomputed.sorted_keywords[i];
114 if (kw.bytes.len != bytes.len)
115 return null;
116 if (mem.eql(u8, kw.bytes, bytes))
117 return kw.id;
118 i += 1;
119 if (i >= precomputed.sorted_keywords.len)
120 return null;
121 }
68 return keywords.get(bytes);
12269 }
12370
12471 pub const Id = enum {