| ... | ... | @@ -13,14 +13,11 @@ pub const Token = struct { |
| 13 | 13 | pub const Keyword = struct { |
| 14 | 14 | bytes: []const u8, |
| 15 | 15 | id: Id, |
| 16 | | hash: u32, |
| 17 | 16 | |
| 18 | 17 | fn init(bytes: []const u8, id: Id) Keyword { |
| 19 | | @setEvalBranchQuota(2000); |
| 20 | 18 | return .{ |
| 21 | 19 | .bytes = bytes, |
| 22 | 20 | .id = id, |
| 23 | | .hash = std.hash_map.hashString(bytes), |
| 24 | 21 | }; |
| 25 | 22 | } |
| 26 | 23 | }; |
| ... | ... | @@ -79,15 +76,49 @@ pub const Token = struct { |
| 79 | 76 | Keyword.init("while", .Keyword_while), |
| 80 | 77 | }; |
| 81 | 78 | |
| 82 | | // TODO perfect hash at comptime |
| 83 | 79 | pub fn getKeyword(bytes: []const u8) ?Id { |
| 84 | | var hash = std.hash_map.hashString(bytes); |
| 85 | | for (keywords) |kw| { |
| 86 | | if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) { |
| 87 | | return kw.id; |
| 80 | const precomputed = comptime blk: { |
| 81 | @setEvalBranchQuota(2000); |
| 82 | var sorted_keywords = keywords; |
| 83 | const lenAsc = (struct { |
| 84 | fn lenAsc(a: Keyword, b: Keyword) bool { |
| 85 | return a.bytes.len < b.bytes.len; |
| 86 | } |
| 87 | }).lenAsc; |
| 88 | std.sort.sort(Keyword, &sorted_keywords, lenAsc); |
| 89 | const min_len = sorted_keywords[0].bytes.len; |
| 90 | const max_len = sorted_keywords[sorted_keywords.len - 1].bytes.len; |
| 91 | var len_indexes: [max_len + 1]usize = undefined; |
| 92 | var len: usize = 0; |
| 93 | var kw_i: usize = 0; |
| 94 | while (len <= max_len) : (len += 1) { |
| 95 | // find the first keyword len == len |
| 96 | while (len > sorted_keywords[kw_i].bytes.len) { |
| 97 | kw_i += 1; |
| 98 | } |
| 99 | len_indexes[len] = kw_i; |
| 88 | 100 | } |
| 101 | break :blk .{ |
| 102 | .min_len = min_len, |
| 103 | .max_len = max_len, |
| 104 | .sorted_keywords = sorted_keywords, |
| 105 | .len_indexes = len_indexes, |
| 106 | }; |
| 107 | }; |
| 108 | if (bytes.len < precomputed.min_len or bytes.len > precomputed.max_len) |
| 109 | return null; |
| 110 | |
| 111 | var i = precomputed.len_indexes[bytes.len]; |
| 112 | while (true) { |
| 113 | const kw = precomputed.sorted_keywords[i]; |
| 114 | if (kw.bytes.len != bytes.len) |
| 115 | return null; |
| 116 | if (mem.eql(u8, kw.bytes, bytes)) |
| 117 | return kw.id; |
| 118 | i += 1; |
| 119 | if (i >= precomputed.sorted_keywords.len) |
| 120 | return null; |
| 89 | 121 | } |
| 90 | | return null; |
| 91 | 122 | } |
| 92 | 123 | |
| 93 | 124 | pub const Id = enum { |
Andrew Kelley <andrew@ziglang.org> 2020-05-26 14:07:47-04:00
lib