authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 02:44:23+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
logd75697a6a3e2c8d96819c365dcb5690d4d8028e9
treebb942b9ceaaad90e5325e9f7039601f41acb137f
parent26bf410b061b9d6d18e4945417ddec62d7486e9c
signaturelock-open Commit is signed but in an unrecognized format.

std-c tokenizer keywords

1 files changed, 184 insertions(+), 10 deletions(-)

lib/std/c/tokenizer.zig+184-10
......@@ -1,5 +1,5 @@
11const std = @import("std");
2const expect = std.testing.expect;
2const mem = std.mem;
33
44pub const Source = struct {
55 buffer: []const u8,
......@@ -7,11 +7,19 @@ pub const Source = struct {
77};
88
99pub const Token = struct {
10 id: union(enum) {
10 id: Id,
11 start: usize,
12 end: usize,
13 source: *Source,
14
15 pub const Id = union(enum) {
1116 Invalid,
1217 Eof,
1318 Nl,
1419 Identifier,
20
21 /// special case for #include <...>
22 MacroString,
1523 StringLiteral: StrKind,
1624 CharLiteral: StrKind,
1725 IntegerLiteral: NumSuffix,
......@@ -68,10 +76,160 @@ pub const Token = struct {
6876 MultiLineComment,
6977 Hash,
7078 HashHash,
71 },
72 start: usize,
73 end: usize,
74 source: *Source,
79
80 Keyword_auto,
81 Keyword_break,
82 Keyword_case,
83 Keyword_char,
84 Keyword_const,
85 Keyword_continue,
86 Keyword_default,
87 Keyword_do,
88 Keyword_double,
89 Keyword_else,
90 Keyword_enum,
91 Keyword_extern,
92 Keyword_float,
93 Keyword_for,
94 Keyword_goto,
95 Keyword_if,
96 Keyword_int,
97 Keyword_long,
98 Keyword_register,
99 Keyword_return,
100 Keyword_short,
101 Keyword_signed,
102 Keyword_sizeof,
103 Keyword_static,
104 Keyword_struct,
105 Keyword_switch,
106 Keyword_typedef,
107 Keyword_union,
108 Keyword_unsigned,
109 Keyword_void,
110 Keyword_volatile,
111 Keyword_while,
112
113 // ISO C99
114 Keyword_bool,
115 Keyword_complex,
116 Keyword_imaginary,
117 Keyword_inline,
118 Keyword_restrict,
119
120 // ISO C11
121 Keyword_alignas,
122 Keyword_alignof,
123 Keyword_atomic,
124 Keyword_generic,
125 Keyword_noreturn,
126 Keyword_static_assert,
127 Keyword_thread_local,
128
129 // Preprocessor
130 Keyword_include,
131 Keyword_define,
132 Keyword_ifdef,
133 Keyword_ifndef,
134 Keyword_error,
135 Keyword_pragma,
136 };
137
138 pub const Keyword = struct {
139 bytes: []const u8,
140 id: Id,
141 hash: u32,
142
143 fn init(bytes: []const u8, id: Id) Keyword {
144 @setEvalBranchQuota(2000);
145 return .{
146 .bytes = bytes,
147 .id = id,
148 .hash = std.hash_map.hashString(bytes),
149 };
150 }
151 };
152
153 // TODO extensions
154 pub const keywords = [_]Keyword{
155 Keyword.init("auto", .Keyword_auto),
156 Keyword.init("break", .Keyword_break),
157 Keyword.init("case", .Keyword_case),
158 Keyword.init("char", .Keyword_char),
159 Keyword.init("const", .Keyword_const),
160 Keyword.init("continue", .Keyword_continue),
161 Keyword.init("default", .Keyword_default),
162 Keyword.init("do", .Keyword_do),
163 Keyword.init("double", .Keyword_double),
164 Keyword.init("else", .Keyword_else),
165 Keyword.init("enum", .Keyword_enum),
166 Keyword.init("extern", .Keyword_extern),
167 Keyword.init("float", .Keyword_float),
168 Keyword.init("for", .Keyword_for),
169 Keyword.init("goto", .Keyword_goto),
170 Keyword.init("if", .Keyword_if),
171 Keyword.init("int", .Keyword_int),
172 Keyword.init("long", .Keyword_long),
173 Keyword.init("register", .Keyword_register),
174 Keyword.init("return", .Keyword_return),
175 Keyword.init("short", .Keyword_short),
176 Keyword.init("signed", .Keyword_signed),
177 Keyword.init("sizeof", .Keyword_sizeof),
178 Keyword.init("static", .Keyword_static),
179 Keyword.init("struct", .Keyword_struct),
180 Keyword.init("switch", .Keyword_switch),
181 Keyword.init("typedef", .Keyword_typedef),
182 Keyword.init("union", .Keyword_union),
183 Keyword.init("unsigned", .Keyword_unsigned),
184 Keyword.init("void", .Keyword_void),
185 Keyword.init("volatile", .Keyword_volatile),
186 Keyword.init("while", .Keyword_while),
187
188 // ISO C99
189 Keyword.init("_Bool", .Keyword_bool),
190 Keyword.init("_Complex", .Keyword_complex),
191 Keyword.init("_Imaginary", .Keyword_imaginary),
192 Keyword.init("inline", .Keyword_inline),
193 Keyword.init("restrict", .Keyword_restrict),
194
195 // ISO C11
196 Keyword.init("_Alignas", .Keyword_alignas),
197 Keyword.init("_Alignof", .Keyword_alignof),
198 Keyword.init("_Atomic", .Keyword_atomic),
199 Keyword.init("_Generic", .Keyword_generic),
200 Keyword.init("_Noreturn", .Keyword_noreturn),
201 Keyword.init("_Static_assert", .Keyword_static_assert),
202 Keyword.init("_Thread_local", .Keyword_thread_local),
203
204 // Preprocessor
205 Keyword.init("include", .Keyword_include),
206 Keyword.init("define", .Keyword_define),
207 Keyword.init("ifdef", .Keyword_ifdef),
208 Keyword.init("ifndef", .Keyword_ifndef),
209 Keyword.init("error", .Keyword_error),
210 Keyword.init("pragma", .Keyword_pragma),
211 };
212
213 // TODO perfect hash at comptime
214 pub fn getKeyword(bytes: []const u8, macro: bool) ?Id {
215 var hash = std.hash_map.hashString(bytes);
216 for (keywords) |kw| {
217 if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) {
218 switch (kw.id) {
219 .Keyword_include,
220 .Keyword_define,
221 .Keyword_ifdef,
222 .Keyword_ifndef,
223 .Keyword_error,
224 .Keyword_pragma,
225 => if (!macro) return null,
226 else => {},
227 }
228 return kw.id;
229 }
230 }
231 return null;
232 }
75233
76234 pub const NumSuffix = enum {
77235 None,
......@@ -95,6 +253,7 @@ pub const Token = struct {
95253pub const Tokenizer = struct {
96254 source: *Source,
97255 index: usize = 0,
256 prev_tok_id: @TagType(Token.Id),
98257
99258 pub fn next(self: *Tokenizer) Token {
100259 const start_index = self.index;
......@@ -124,6 +283,9 @@ pub const Tokenizer = struct {
124283 Percent,
125284 Asterisk,
126285 Plus,
286
287 /// special case for #include <...>
288 MacroString,
127289 AngleBracketLeft,
128290 AngleBracketAngleBracketLeft,
129291 AngleBracketRight,
......@@ -189,7 +351,6 @@ pub const Tokenizer = struct {
189351 },
190352 'a'...'t', 'v'...'z', 'A'...'K', 'M'...'T', 'V'...'Z', '_' => {
191353 state = .Identifier;
192 result.id = .Identifier;
193354 },
194355 '=' => {
195356 state = .Equal;
......@@ -250,7 +411,10 @@ pub const Tokenizer = struct {
250411 state = .Plus;
251412 },
252413 '<' => {
253 state = .AngleBracketLeft;
414 if (self.prev_tok_id == .Keyword_include)
415 state = .MacroString
416 else
417 state = .AngleBracketLeft;
254418 },
255419 '>' => {
256420 state = .AngleBracketRight;
......@@ -442,7 +606,7 @@ pub const Tokenizer = struct {
442606 .Identifier => switch (c) {
443607 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
444608 else => {
445 result.id = .Identifier;
609 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash) orelse .Identifier;
446610 break;
447611 },
448612 },
......@@ -522,6 +686,14 @@ pub const Tokenizer = struct {
522686 break;
523687 },
524688 },
689 .MacroString => switch (c) {
690 '>' => {
691 result.id = .MacroString;
692 self.index += 1;
693 break;
694 },
695 else => {},
696 },
525697 .AngleBracketLeft => switch (c) {
526698 '<' => {
527699 state = .AngleBracketAngleBracketLeft;
......@@ -859,7 +1031,7 @@ pub const Tokenizer = struct {
8591031 switch (state) {
8601032 .Start => {},
8611033 .u, .u8, .U, .L, .Identifier => {
862 result.id = .Identifier;
1034 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash) orelse .Identifier;
8631035 },
8641036
8651037 .Cr,
......@@ -876,6 +1048,7 @@ pub const Tokenizer = struct {
8761048 .FloatFractionHex,
8771049 .FloatExponent,
8781050 .FloatExponentDigits,
1051 .MacroString,
8791052 => result.id = .Invalid,
8801053
8811054 .IntegerLiteralOct,
......@@ -910,6 +1083,7 @@ pub const Tokenizer = struct {
9101083 }
9111084 }
9121085
1086 self.prev_tok_id = result.id;
9131087 result.end = self.index;
9141088 return result;
9151089 }