authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2020-05-26 21:36:58-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2020-05-26 21:36:58-07:00
loga9dd79d29359760f46d4a28c4dcf65b720b7b755
tree9d248adb9c66d8e6772255029ee3f949b46274fa
parent0865e5d360500e101fbf5e23b842bb6e26435a38

Use std.ComptimeStringMap in zig's tokenizer

Should have no performance change, the implementation was just made generic/re-usable

1 files changed, 54 insertions(+), 107 deletions(-)

lib/std/zig/tokenizer.zig+54-107
...@@ -10,115 +10,62 @@ pub const Token = struct {...@@ -10,115 +10,62 @@ pub const Token = struct {
10 end: usize,10 end: usize,
11 };11 };
1212
13 pub const Keyword = struct {13 pub const keywords = std.ComptimeStringMap(Id, .{
14 bytes: []const u8,14 .{"align", .Keyword_align},
15 id: Id,15 .{"allowzero", .Keyword_allowzero},
1616 .{"and", .Keyword_and},
17 fn init(bytes: []const u8, id: Id) Keyword {17 .{"anyframe", .Keyword_anyframe},
18 return .{18 .{"asm", .Keyword_asm},
19 .bytes = bytes,19 .{"async", .Keyword_async},
20 .id = id,20 .{"await", .Keyword_await},
21 };21 .{"break", .Keyword_break},
22 }22 .{"callconv", .Keyword_callconv},
23 };23 .{"catch", .Keyword_catch},
2424 .{"comptime", .Keyword_comptime},
25 pub const keywords = [_]Keyword{25 .{"const", .Keyword_const},
26 Keyword.init("align", .Keyword_align),26 .{"continue", .Keyword_continue},
27 Keyword.init("allowzero", .Keyword_allowzero),27 .{"defer", .Keyword_defer},
28 Keyword.init("and", .Keyword_and),28 .{"else", .Keyword_else},
29 Keyword.init("anyframe", .Keyword_anyframe),29 .{"enum", .Keyword_enum},
30 Keyword.init("asm", .Keyword_asm),30 .{"errdefer", .Keyword_errdefer},
31 Keyword.init("async", .Keyword_async),31 .{"error", .Keyword_error},
32 Keyword.init("await", .Keyword_await),32 .{"export", .Keyword_export},
33 Keyword.init("break", .Keyword_break),33 .{"extern", .Keyword_extern},
34 Keyword.init("callconv", .Keyword_callconv),34 .{"false", .Keyword_false},
35 Keyword.init("catch", .Keyword_catch),35 .{"fn", .Keyword_fn},
36 Keyword.init("comptime", .Keyword_comptime),36 .{"for", .Keyword_for},
37 Keyword.init("const", .Keyword_const),37 .{"if", .Keyword_if},
38 Keyword.init("continue", .Keyword_continue),38 .{"inline", .Keyword_inline},
39 Keyword.init("defer", .Keyword_defer),39 .{"noalias", .Keyword_noalias},
40 Keyword.init("else", .Keyword_else),40 .{"noasync", .Keyword_nosuspend}, // TODO: remove this
41 Keyword.init("enum", .Keyword_enum),41 .{"noinline", .Keyword_noinline},
42 Keyword.init("errdefer", .Keyword_errdefer),42 .{"nosuspend", .Keyword_nosuspend},
43 Keyword.init("error", .Keyword_error),43 .{"null", .Keyword_null},
44 Keyword.init("export", .Keyword_export),44 .{"or", .Keyword_or},
45 Keyword.init("extern", .Keyword_extern),45 .{"orelse", .Keyword_orelse},
46 Keyword.init("false", .Keyword_false),46 .{"packed", .Keyword_packed},
47 Keyword.init("fn", .Keyword_fn),47 .{"pub", .Keyword_pub},
48 Keyword.init("for", .Keyword_for),48 .{"resume", .Keyword_resume},
49 Keyword.init("if", .Keyword_if),49 .{"return", .Keyword_return},
50 Keyword.init("inline", .Keyword_inline),50 .{"linksection", .Keyword_linksection},
51 Keyword.init("noalias", .Keyword_noalias),51 .{"struct", .Keyword_struct},
52 Keyword.init("noasync", .Keyword_nosuspend), // TODO: remove this52 .{"suspend", .Keyword_suspend},
53 Keyword.init("noinline", .Keyword_noinline),53 .{"switch", .Keyword_switch},
54 Keyword.init("nosuspend", .Keyword_nosuspend),54 .{"test", .Keyword_test},
55 Keyword.init("null", .Keyword_null),55 .{"threadlocal", .Keyword_threadlocal},
56 Keyword.init("or", .Keyword_or),56 .{"true", .Keyword_true},
57 Keyword.init("orelse", .Keyword_orelse),57 .{"try", .Keyword_try},
58 Keyword.init("packed", .Keyword_packed),58 .{"undefined", .Keyword_undefined},
59 Keyword.init("pub", .Keyword_pub),59 .{"union", .Keyword_union},
60 Keyword.init("resume", .Keyword_resume),60 .{"unreachable", .Keyword_unreachable},
61 Keyword.init("return", .Keyword_return),61 .{"usingnamespace", .Keyword_usingnamespace},
62 Keyword.init("linksection", .Keyword_linksection),62 .{"var", .Keyword_var},
63 Keyword.init("struct", .Keyword_struct),63 .{"volatile", .Keyword_volatile},
64 Keyword.init("suspend", .Keyword_suspend),64 .{"while", .Keyword_while},
65 Keyword.init("switch", .Keyword_switch),65 });
66 Keyword.init("test", .Keyword_test),
67 Keyword.init("threadlocal", .Keyword_threadlocal),
68 Keyword.init("true", .Keyword_true),
69 Keyword.init("try", .Keyword_try),
70 Keyword.init("undefined", .Keyword_undefined),
71 Keyword.init("union", .Keyword_union),
72 Keyword.init("unreachable", .Keyword_unreachable),
73 Keyword.init("usingnamespace", .Keyword_usingnamespace),
74 Keyword.init("var", .Keyword_var),
75 Keyword.init("volatile", .Keyword_volatile),
76 Keyword.init("while", .Keyword_while),
77 };
7866
79 pub fn getKeyword(bytes: []const u8) ?Id {67 pub fn getKeyword(bytes: []const u8) ?Id {
80 const precomputed = comptime blk: {68 return keywords.get(bytes);
81 @setEvalBranchQuota(2000);
82 var sorted_keywords = keywords;
83 const lenAsc = (struct {
84 fn lenAsc(a: Keyword, b: Keyword) bool {
85 return a.bytes.len < b.bytes.len;
86 }
87 }).lenAsc;
88 std.sort.sort(Keyword, &sorted_keywords, lenAsc);
89 const min_len = sorted_keywords[0].bytes.len;
90 const max_len = sorted_keywords[sorted_keywords.len - 1].bytes.len;
91 var len_indexes: [max_len + 1]usize = undefined;
92 var len: usize = 0;
93 var kw_i: usize = 0;
94 while (len <= max_len) : (len += 1) {
95 // find the first keyword len == len
96 while (len > sorted_keywords[kw_i].bytes.len) {
97 kw_i += 1;
98 }
99 len_indexes[len] = kw_i;
100 }
101 break :blk .{
102 .min_len = min_len,
103 .max_len = max_len,
104 .sorted_keywords = sorted_keywords,
105 .len_indexes = len_indexes,
106 };
107 };
108 if (bytes.len < precomputed.min_len or bytes.len > precomputed.max_len)
109 return null;
110
111 var i = precomputed.len_indexes[bytes.len];
112 while (true) {
113 const kw = precomputed.sorted_keywords[i];
114 if (kw.bytes.len != bytes.len)
115 return null;
116 if (mem.eql(u8, kw.bytes, bytes))
117 return kw.id;
118 i += 1;
119 if (i >= precomputed.sorted_keywords.len)
120 return null;
121 }
122 }69 }
12370
124 pub const Id = enum {71 pub const Id = enum {