authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-15 10:30:38+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-15 10:30:38+02:00
logc3724a6e723dfb5ec78c6ca87e2f02e121d39bc2
treefd77036cd4341bba7f07853bb808e32acf8f80ec
parente57e3602e7802789d56a076c6052092715997c24
signature Commit is signed but in an unrecognized format.

translate-c-2 c tokenizer

1 files changed, 458 insertions(+), 0 deletions(-)

src-self-hosted/c_tokenizer.zig created+458
...@@ -0,0 +1,458 @@
1const std = @import("std");
2
3pub const TokenList = std.SegmentedList(CToken, 32);
4
5pub const CToken = struct {
6 id: Id,
7 bytes: []const u8,
8 num_lit_suffix: NumLitSuffix = undefined,
9
10 pub const Id = enum {
11 CharLit,
12 StrLit,
13 NumLitInt,
14 NumLitFloat,
15 Identifier,
16 Minus,
17 Slash,
18 LParen,
19 RParen,
20 Eof,
21 Dot,
22 Asterisk,
23 Bang,
24 Tilde,
25 Shl,
26 Lt,
27 };
28
29 pub const NumLitSuffix = enum {
30 None,
31 L,
32 U,
33 LU,
34 LL,
35 LLU,
36 };
37};
38
39pub fn tokenizeCMacro(tl: *TokenList, chars: [*]const u8) !void {
40 var index: usize = 0;
41 while (true) {
42 const tok = try next(chars[index..], &index);
43 tl.push(tok);
44 if (tok.id == .Eof)
45 return;
46 }
47}
48
49fn next(chars: [*]const u8, index: *usize) !CToken {
50 var state: enum {
51 Start,
52 GotLt,
53 ExpectChar,
54 ExpectEndQuot,
55 OpenComment,
56 Comment,
57 CommentStar,
58 Backslash,
59 String,
60 Identifier,
61 Decimal,
62 Octal,
63 GotZero,
64 Hex,
65 Float,
66 ExpSign,
67 FloatExp,
68 FloatExpFirst,
69 NumLitIntSuffixU,
70 NumLitIntSuffixL,
71 NumLitIntSuffixLL,
72 NumLitIntSuffixUL,
73 GotLt,
74 } = .Start;
75
76 var result = CToken{
77 .bytes = "",
78 .id = .Eof,
79 };
80 var begin_index: usize = 0;
81 var digits: u8 = 0;
82 var pre_escape = .Start;
83
84 for (chars[begin_index..]) |c, i| {
85 if (c == 0) {
86 switch (state) {
87 .Start => {
88 return result;
89 },
90 .Identifier,
91 .Decimal,
92 .Hex,
93 .Octal,
94 .GotZero,
95 .NumLitIntSuffixU,
96 .NumLitIntSuffixL,
97 .NumLitIntSuffixUL,
98 .NumLitIntSuffixLL,
99 .Float,
100 .FloatExp,
101 .GotLt,
102 => {
103 return result;
104 },
105 .ExpectChar,
106 .ExpectEndQuot,
107 .OpenComment,
108 .LineComment,
109 .Comment,
110 .CommentStar,
111 .Backslash,
112 .String,
113 .ExpSign,
114 .FloatExpFirst,
115 => return error.TokenizingFailed,
116 }
117 }
118 index.* += 1;
119 switch (state) {
120 .Start => {
121 switch (c) {
122 ' ', '\t', '\x0B', '\x0C' => {},
123 '\'' => {
124 state = .ExpectChar;
125 result.id = .CharLit;
126 begin_index = i;
127 },
128 '\"' => {
129 state = .String;
130 result.id = .StrLit;
131 begin_index = i;
132 },
133 '/' => {
134 state = .OpenComment;
135 },
136 '\\' => {
137 state = .Backslash;
138 },
139 '\n', '\r' => {
140 return result;
141 },
142 'a'...'z', 'A'...'Z', '_' => {
143 state = .Identifier;
144 result.id = .Identifier;
145 begin_index = i;
146 },
147 '1'...'9' => {
148 state = .Decimal;
149 result.id = .NumLitInt;
150 begin_index = i;
151 },
152 '0' => {
153 state = .GotZero;
154 result.id = .NumLitInt;
155 begin_index = i;
156 },
157 '.' => {
158 result.id = .Dot;
159 return result;
160 },
161 '<' => {
162 result.id = .Lt;
163 state = .GotLt;
164 },
165 '(' => {
166 result.id = .LParen;
167 return result;
168 },
169 ')' => {
170 result.id = .RParen;
171 return result;
172 },
173 '*' => {
174 result.id = .Asterisk;
175 return result;
176 },
177 '-' => {
178 result.id = .Minus;
179 return result;
180 },
181 '!' => {
182 result.id = .Bang;
183 return result;
184 },
185 '~' => {
186 result.id = .Tilde;
187 return result;
188 },
189 else => return error.TokenizingFailed,
190 }
191 },
192 .GotLt => {
193 switch (c) {
194 '<' => {
195 result.id = .Shl;
196 return result;
197 },
198 else => {
199 return result;
200 },
201 }
202 },
203 .Float => {
204 switch (c) {
205 '.', '0'...'9' => {},
206 'e', 'E' => {
207 state = .ExpSign;
208 },
209 'f', 'F', 'l', 'L' => {
210 result.bytes = chars[begin_index..i];
211 return result;
212 },
213 else => {
214 result.bytes = chars[begin_index..i];
215 return result;
216 },
217 }
218 },
219 .ExpSign => {
220 switch (c) {
221 '+', '-' => {
222 state = .FloatExpFirst;
223 },
224 '0'...'9' => {
225 state = .FloatExp;
226 },
227 else => return error.TokenizingFailed,
228 }
229 },
230 .FloatExpFirst => {
231 switch (c) {
232 '0'...'9' => {
233 state = .FloatExp;
234 },
235 else => return error.TokenizingFailed,
236 }
237 },
238 .FloatExp => {
239 switch (c) {
240 '0'...'9' => {},
241 'f', 'F', 'l', 'L' => {
242 result.bytes = chars[begin_index..i];
243 return result;
244 },
245 else => {
246 result.bytes = chars[begin_index..i];
247 return result;
248 },
249 }
250 },
251 .Decimal => {
252 switch (c) {
253 '0'...'9' => {},
254 '\'' => {},
255 'u', 'U' => {
256 state = .NumLitIntSuffixU;
257 result.num_lit_suffix = .U;
258 },
259 'l', 'L' => {
260 state = .NumLitIntSuffixL;
261 result.num_lit_suffix = .L;
262 },
263 '.' => {
264 result.id = .NumLitFloat;
265 state = .Float;
266 },
267 else => {
268 result.bytes = chars[begin_index..i];
269 return result;
270 },
271 }
272 },
273 .GotZero => {
274 switch (c) {
275 'x', 'X' => {
276 state = .Hex;
277 },
278 '.' => {
279 state = .Float;
280 result.id = .NumLitFloat;
281 },
282 'l', 'L', 'u', 'U' => {
283 c -= 1;
284 state = .Decimal;
285 },
286 else => {
287 state = .Octal;
288 },
289 }
290 },
291 .Octal => {
292 switch (c) {
293 '0'...'7' => {},
294 '8', '9' => return error.TokenizingFailed,
295 else => {
296 result.bytes = chars[begin_index..i];
297 return result;
298 },
299 }
300 },
301 .Hex => {
302 switch (c) {
303 '0'...'9', 'a'...'f', 'A'...'F' => {},
304
305 'p', 'P' => {
306 result.id = .NumLitFloat;
307 state = .ExpSign;
308 },
309 'u', 'U' => {
310 // marks the number literal as unsigned
311 state = .NumLitIntSuffixU;
312 result.num_lit_suffix = .U;
313 },
314 'l', 'L' => {
315 // marks the number literal as long
316 state = .NumLitIntSuffixL;
317 result.num_lit_suffix = .L;
318 },
319 else => {
320 result.bytes = chars[begin_index..i];
321 return result;
322 },
323 }
324 },
325 .NumLitIntSuffixU => {
326 switch (c) {
327 'l', 'L' => {
328 result.num_lit_suffix = .LU;
329 state = .NumLitIntSuffixUL;
330 },
331 else => {
332 result.bytes = chars[begin_index..i - 1];
333 return result;
334 },
335 }
336 },
337 .NumLitIntSuffixL => {
338 switch (c) {
339 'l', 'L' => {
340 result.num_lit_suffix = .LL;
341 state = .NumLitIntSuffixLL;
342 },
343 'u', 'U' => {
344 result.num_lit_suffix = .LU;
345 result.bytes = chars[begin_index..i - 2];
346 return result;
347 },
348 else => {
349 result.bytes = chars[begin_index..i - 1];
350 return result;
351 },
352 }
353 },
354 .NumLitIntSuffixLL => {
355 switch (c) {
356 'u', 'U' => {
357 result.num_lit_suffix = .LLU;
358 result.bytes = chars[begin_index..i - 3];
359 return result;
360 },
361 else => {
362 result.bytes = chars[begin_index..i - 2];
363 return result;
364 },
365 }
366 },
367 .NumLitIntSuffixUL => {
368 switch (c) {
369 'l', 'L' => {
370 result.num_lit_suffix = .LLU;
371 result.bytes = chars[begin_index..i - 3];
372 return result;
373 },
374 else => {
375 result.bytes = chars[begin_index..i - 2];
376 return result;
377 },
378 }
379 },
380 .Identifier => {
381 switch (c) {
382 '_', 'a'...'z', 'A'...'Z', '0'...'9' => {},
383 else => {
384 result.bytes = chars[begin_index..i];
385 return result;
386 },
387 }
388 },
389 .String => {
390 switch (c) {
391 '\"' => {
392 result.bytes = chars[begin_index + 1 .. i];
393 return result;
394 },
395 else => {},
396 }
397 },
398 .ExpectChar => {
399 switch (c) {
400 '\'' => return error.TokenizingFailed,
401 else => {
402 state = .ExpectEndQuot;
403 },
404 }
405 },
406 .ExpectEndQuot => {
407 switch (c) {
408 '\'' => {
409 result.bytes = chars[begin_index + 1 .. i];
410 return result;
411 },
412 else => return error.TokenizingFailed,
413 }
414 },
415 .OpenComment => {
416 switch (c) {
417 '/' => {
418 return result;
419 },
420 '*' => {
421 state = .Comment;
422 },
423 else => {
424 result.id = .Slash;
425 return result;
426 },
427 }
428 },
429 .Comment => {
430 switch (c) {
431 '*' => {
432 state = .CommentStar;
433 },
434 else => {},
435 }
436 },
437 .CommentStar => {
438 switch (c) {
439 '/' => {
440 state = .Start;
441 },
442 else => {
443 state = .Comment;
444 },
445 }
446 },
447 .Backslash => {
448 switch (c) {
449 ' ', '\t', '\x0B', '\x0C' => {},
450 '\n', '\r' => {
451 state = .Start;
452 },
453 else => return error.TokenizingFailed,
454 }
455 },
456 }
457 }
458}