1const std = @import("std");
2const assert = std.debug.assert;
3const Source = @import("Preprocessor.zig").Source;
4
5const Tokenizer = @This();
6
7pub fn init(buf: []const u8, source: Source.Id) Tokenizer {
8 return .{ .buf = buf, .source = source };
9}
10
11buf: []const u8,
12index: u32 = 0,
13source: Source.Id,
14
15pub const Token = struct {
16 pub const Id = enum {
17 bang,
18 eof,
19 equal_equal,
20 hash,
21 hash_hash,
22 macro_param,
23 identifier,
24 keyword_if,
25 keyword_ifndef,
26 keyword_ifdef,
27 keyword_define,
28 keyword_endif,
29 keyword_defined,
30 keyword_include,
31 keyword_elif,
32 keyword_else,
33 keyword_undef,
34 keyword_error,
35 l_paren,
36 nl,
37 pp_num,
38 pipe_pipe,
39 r_paren,
40 semicolon,
41 string_literal,
42 whitespace,
43 one,
44 zero,
45
46 pub fn isInfix(id: Id) bool {
47 switch (id) {
48 .pipe_pipe, .equal_equal => return true,
49 else => return false,
50 }
51 }
52
53 pub fn isMacroIdentifier(id: Id) bool {
54 switch (id) {
55 .keyword_if,
56 .keyword_ifndef,
57 .keyword_ifdef,
58 .keyword_define,
59 .keyword_endif,
60 .keyword_defined,
61 .keyword_include,
62 .keyword_elif,
63 .keyword_else,
64 .keyword_undef,
65 .keyword_error,
66 .identifier,
67 => return true,
68 else => return false,
69 }
70 }
71 };
72
73 const all_kws = std.StaticStringMap(Id).initComptime(.{
74 .{ "define", .keyword_define },
75 .{ "defined", .keyword_defined },
76 .{ "else", .keyword_else },
77 .{ "endif", .keyword_endif },
78 .{ "if", .keyword_if },
79 .{ "elif", .keyword_elif },
80 .{ "ifdef", .keyword_ifdef },
81 .{ "ifndef", .keyword_ifndef },
82 .{ "include", .keyword_include },
83 .{ "undef", .keyword_undef },
84 .{ "error", .keyword_error },
85 });
86
87 id: Id,
88 source: Source.Id,
89 start: u32 = 0,
90 end: u32 = 0,
91
92 fn getTokenId(str: []const u8) Id {
93 return all_kws.get(str) orelse .identifier;
94 }
95};
96
97pub fn next(self: *Tokenizer) Token {
98 var state: enum {
99 start,
100 cr,
101 string_literal,
102 identifier,
103 equal,
104 slash,
105 line_comment,
106 hash,
107 pipe,
108 pp_num,
109 } = .start;
110
111 const start = self.index;
112 var id: Token.Id = .eof;
113
114 while (self.index < self.buf.len) : (self.index += 1) {
115 const c = self.buf[self.index];
116 switch (state) {
117 .start => switch (c) {
118 '\r' => {
119 id = .nl;
120 state = .cr;
121 },
122 '\n' => {
123 id = .nl;
124 self.index += 1;
125 break;
126 },
127 '!' => {
128 id = .bang;
129 self.index += 1;
130 break;
131 },
132 '"' => {
133 id = .string_literal;
134 state = .string_literal;
135 },
136 '|' => state = .pipe,
137 '=' => state = .equal,
138 '(' => {
139 id = .l_paren;
140 self.index += 1;
141 break;
142 },
143 ')' => {
144 id = .r_paren;
145 self.index += 1;
146 break;
147 },
148 ';' => {
149 id = .semicolon;
150 self.index += 1;
151 break;
152 },
153 '/' => state = .slash,
154 '#' => state = .hash,
155 '0'...'9' => state = .pp_num,
156 ' ' => {
157 id = .whitespace;
158 self.index += 1;
159 break;
160 },
161 else => state = .identifier,
162 },
163 .cr => switch (c) {
164 '\n' => {
165 self.index += 1;
166 break;
167 },
168 else => break,
169 },
170 .pipe => switch (c) {
171 '|' => {
172 id = .pipe_pipe;
173 self.index += 1;
174 break;
175 },
176 else => unreachable,
177 },
178 .hash => switch (c) {
179 '#' => {
180 id = .hash_hash;
181 self.index += 1;
182 break;
183 },
184 else => {
185 id = .hash;
186 break;
187 },
188 },
189 .string_literal => switch (c) {
190 '"' => {
191 self.index += 1;
192 break;
193 },
194 else => {},
195 },
196 .identifier => switch (c) {
197 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
198 else => {
199 id = Token.getTokenId(self.buf[start..self.index]);
200 break;
201 },
202 },
203 .equal => switch (c) {
204 '=' => {
205 id = .equal_equal;
206 self.index += 1;
207 break;
208 },
209 else => unreachable,
210 },
211 .slash => switch (c) {
212 '/' => state = .line_comment,
213 else => {
214 id = .identifier;
215 break;
216 },
217 },
218 .line_comment => switch (c) {
219 '\n' => {
220 self.index -= 1;
221 state = .start;
222 },
223 else => {},
224 },
225 .pp_num => switch (c) {
226 '0'...'9' => {},
227 else => {
228 id = .pp_num;
229 break;
230 },
231 },
232 }
233 } else if (self.index == self.buf.len) {
234 switch (state) {
235 .start, .line_comment, .cr => {},
236 .identifier => id = Token.getTokenId(self.buf[start..self.index]),
237 .hash => id = .hash,
238 .pp_num => id = .pp_num,
239 else => unreachable,
240 }
241 }
242
243 return .{
244 .id = id,
245 .start = start,
246 .end = self.index,
247 .source = self.source,
248 };
249}
250
251pub fn nextNoWS(self: *Tokenizer) Token {
252 var tok = self.next();
253 while (tok.id == .whitespace) tok = self.next();
254 return tok;
255}
256
257fn expectToken(expected: Token.Id, actual: Token) !void {
258 try std.testing.expectEqual(expected, actual.id);
259}
260
261fn testToken(buf: []const u8, expected: Token.Id) !void {
262 var tokenizer = Tokenizer.init(buf, Source.generated);
263 const t = tokenizer.next();
264 try expectToken(expected, t);
265 try expectToken(.eof, tokenizer.next());
266}
267
268test "tokens" {
269 try testToken("TEST", .identifier);
270 try testToken("__x86_64__", .identifier);
271 try testToken("122", .pp_num);
272 try testToken("==", .equal_equal);
273 try testToken("#", .hash);
274 try testToken("##", .hash_hash);
275 try testToken("undef", .keyword_undef);
276 try testToken("||", .pipe_pipe);
277 try testToken("!", .bang);
278 try testToken("else", .keyword_else);
279 try testToken("endif", .keyword_endif);
280 try testToken("include", .keyword_include);
281 try testToken("define", .keyword_define);
282 try testToken("defined", .keyword_defined);
283 try testToken("if", .keyword_if);
284 try testToken("ifdef", .keyword_ifdef);
285 try testToken("ifndef", .keyword_ifndef);
286 try testToken("(", .l_paren);
287 try testToken("\n", .nl);
288 try testToken("\r", .nl);
289 try testToken("\r\n", .nl);
290 try testToken("5", .pp_num);
291 try testToken(")", .r_paren);
292 try testToken("\"str\"", .string_literal);
293 try testToken(" ", .whitespace);
294}
295
296fn expectTokens(contents: []const u8, expected_tokens: []const Token.Id) !void {
297 var tokenizer: Tokenizer = .init(contents, Source.generated);
298 var i: usize = 0;
299 while (i < expected_tokens.len) {
300 const token = tokenizer.next();
301 if (token.id == .whitespace) continue;
302 const expected_token_id = expected_tokens[i];
303 i += 1;
304 if (!std.meta.eql(token.id, expected_token_id)) {
305 std.debug.print("expected {s}, found {s}\n", .{ @tagName(expected_token_id), @tagName(token.id) });
306 return error.TokensDoNotEqual;
307 }
308 }
309 const last_token = tokenizer.next();
310 try std.testing.expect(last_token.id == .eof);
311}
312
313test "preprocessor keywords" {
314 try expectTokens(
315 \\#if
316 \\#ifndef
317 \\#ifdef
318 \\#define
319 \\#endif
320 \\defined
321 \\#include
322 \\#elif
323 \\#else
324 \\#undef
325 \\#error
326 , &.{
327 .hash,
328 .keyword_if,
329 .nl,
330 .hash,
331 .keyword_ifndef,
332 .nl,
333 .hash,
334 .keyword_ifdef,
335 .nl,
336 .hash,
337 .keyword_define,
338 .nl,
339 .hash,
340 .keyword_endif,
341 .nl,
342 .keyword_defined,
343 .nl,
344 .hash,
345 .keyword_include,
346 .nl,
347 .hash,
348 .keyword_elif,
349 .nl,
350 .hash,
351 .keyword_else,
352 .nl,
353 .hash,
354 .keyword_undef,
355 .nl,
356 .hash,
357 .keyword_error,
358 });
359}