authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-21 15:48:13-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-21 15:48:13-07:00
log35362f8137b2c5109e6bc39cb12048c016b5b580
tree7c0506d8fdb92341cb412ee984288e23c98187b6
parenta380b803ac8b4eefcde4d3d552cdcbc8010aa798

better parsing of C macros

See #88

5 files changed, 804 insertions(+), 204 deletions(-)

CMakeLists.txt+1
......@@ -40,6 +40,7 @@ set(ZIG_SOURCES
4040 "${CMAKE_SOURCE_DIR}/src/ast_render.cpp"
4141 "${CMAKE_SOURCE_DIR}/src/bignum.cpp"
4242 "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp"
43 "${CMAKE_SOURCE_DIR}/src/c_tokenizer.cpp"
4344 "${CMAKE_SOURCE_DIR}/src/parser.cpp"
4445 "${CMAKE_SOURCE_DIR}/src/eval.cpp"
4546 "${CMAKE_SOURCE_DIR}/src/analyze.cpp"
src/c_tokenizer.cpp created+651
......@@ -0,0 +1,651 @@
1/*
2 * Copyright (c) 2016 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8#include "c_tokenizer.hpp"
9#include <inttypes.h>
10
11#define WHITESPACE_EXCEPT_N \
12 ' ': \
13 case '\t': \
14 case '\v': \
15 case '\f'
16
17#define DIGIT_NON_ZERO \
18 '1': \
19 case '2': \
20 case '3': \
21 case '4': \
22 case '5': \
23 case '6': \
24 case '7': \
25 case '8': \
26 case '9'
27
28#define DIGIT \
29 '0': \
30 case DIGIT_NON_ZERO
31
32#define ALPHA \
33 'a': \
34 case 'b': \
35 case 'c': \
36 case 'd': \
37 case 'e': \
38 case 'f': \
39 case 'g': \
40 case 'h': \
41 case 'i': \
42 case 'j': \
43 case 'k': \
44 case 'l': \
45 case 'm': \
46 case 'n': \
47 case 'o': \
48 case 'p': \
49 case 'q': \
50 case 'r': \
51 case 's': \
52 case 't': \
53 case 'u': \
54 case 'v': \
55 case 'w': \
56 case 'x': \
57 case 'y': \
58 case 'z': \
59 case 'A': \
60 case 'B': \
61 case 'C': \
62 case 'D': \
63 case 'E': \
64 case 'F': \
65 case 'G': \
66 case 'H': \
67 case 'I': \
68 case 'J': \
69 case 'K': \
70 case 'L': \
71 case 'M': \
72 case 'N': \
73 case 'O': \
74 case 'P': \
75 case 'Q': \
76 case 'R': \
77 case 'S': \
78 case 'T': \
79 case 'U': \
80 case 'V': \
81 case 'W': \
82 case 'X': \
83 case 'Y': \
84 case 'Z'
85
86#define IDENT_START \
87 ALPHA: \
88 case '_'
89
90#define IDENT \
91 IDENT_START: \
92 case DIGIT
93
94
95static void begin_token(CTokenize *ctok, CTokId id) {
96 assert(ctok->cur_tok == nullptr);
97 ctok->tokens.add_one();
98 ctok->cur_tok = &ctok->tokens.last();
99 ctok->cur_tok->id = id;
100
101 switch (id) {
102 case CTokIdStrLit:
103 memset(&ctok->cur_tok->data.str_lit, 0, sizeof(Buf));
104 buf_resize(&ctok->cur_tok->data.str_lit, 0);
105 break;
106 case CTokIdSymbol:
107 memset(&ctok->cur_tok->data.symbol, 0, sizeof(Buf));
108 buf_resize(&ctok->cur_tok->data.symbol, 0);
109 break;
110 case CTokIdCharLit:
111 case CTokIdNumLitInt:
112 case CTokIdNumLitFloat:
113 case CTokIdMinus:
114 break;
115 }
116}
117
118static void end_token(CTokenize *ctok) {
119 ctok->cur_tok = nullptr;
120}
121
122static void mark_error(CTokenize *ctok) {
123 ctok->error = true;
124}
125
126static void add_char(CTokenize *ctok, uint8_t c) {
127 assert(ctok->cur_tok);
128 if (ctok->cur_tok->id == CTokIdCharLit) {
129 ctok->cur_tok->data.char_lit = c;
130 ctok->state = CTokStateExpectEndQuot;
131 } else if (ctok->cur_tok->id == CTokIdStrLit) {
132 buf_append_char(&ctok->cur_tok->data.str_lit, c);
133 ctok->state = CTokStateString;
134 } else {
135 zig_unreachable();
136 }
137}
138
139static void hex_digit(CTokenize *ctok, uint8_t value) {
140 // TODO @mul_with_overflow
141 ctok->cur_tok->data.num_lit_int *= 16;
142 // TODO @add_with_overflow
143 ctok->cur_tok->data.num_lit_int += value;
144
145 static const uint8_t hex_digit[] = "0123456789abcdef";
146 buf_append_char(&ctok->buf, hex_digit[value]);
147}
148
149static void end_float(CTokenize *ctok) {
150 // TODO detect errors, overflow, and underflow
151 double value = strtod(buf_ptr(&ctok->buf), nullptr);
152
153 ctok->cur_tok->data.num_lit_float = value;
154
155 end_token(ctok);
156 ctok->state = CTokStateStart;
157
158}
159
160void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
161 ctok->tokens.resize(0);
162 ctok->state = CTokStateStart;
163 ctok->error = false;
164 ctok->cur_tok = nullptr;
165
166 buf_resize(&ctok->buf, 0);
167
168 for (; *c; c += 1) {
169 switch (ctok->state) {
170 case CTokStateStart:
171 switch (*c) {
172 case WHITESPACE_EXCEPT_N:
173 break;
174 case '\'':
175 ctok->state = CTokStateExpectChar;
176 begin_token(ctok, CTokIdCharLit);
177 break;
178 case '\"':
179 ctok->state = CTokStateString;
180 begin_token(ctok, CTokIdStrLit);
181 break;
182 case '/':
183 ctok->state = CTokStateOpenComment;
184 break;
185 case '\\':
186 ctok->state = CTokStateBackslash;
187 break;
188 case '\n':
189 goto found_end_of_macro;
190 case IDENT_START:
191 ctok->state = CTokStateIdentifier;
192 begin_token(ctok, CTokIdSymbol);
193 buf_append_char(&ctok->cur_tok->data.symbol, *c);
194 break;
195 case DIGIT_NON_ZERO:
196 ctok->state = CTokStateDecimal;
197 ctok->unsigned_suffix = false;
198 ctok->long_suffix = false;
199 begin_token(ctok, CTokIdNumLitInt);
200 ctok->cur_tok->data.num_lit_int = *c - '0';
201 buf_resize(&ctok->buf, 0);
202 buf_append_char(&ctok->buf, *c);
203 break;
204 case '0':
205 ctok->state = CTokStateGotZero;
206 ctok->unsigned_suffix = false;
207 ctok->long_suffix = false;
208 begin_token(ctok, CTokIdNumLitInt);
209 ctok->cur_tok->data.num_lit_int = 0;
210 buf_resize(&ctok->buf, 0);
211 buf_append_char(&ctok->buf, '0');
212 break;
213 case '.':
214 begin_token(ctok, CTokIdNumLitFloat);
215 ctok->state = CTokStateFloat;
216 buf_init_from_str(&ctok->buf, "0.");
217 break;
218 default:
219 return mark_error(ctok);
220 }
221 break;
222 case CTokStateFloat:
223 switch (*c) {
224 case 'e':
225 case 'E':
226 buf_append_char(&ctok->buf, 'e');
227 ctok->state = CTokStateExpSign;
228 break;
229 case 'f':
230 case 'F':
231 case 'l':
232 case 'L':
233 end_float(ctok);
234 break;
235 case DIGIT:
236 buf_append_char(&ctok->buf, *c);
237 break;
238 default:
239 c -= 1;
240 end_float(ctok);
241 continue;
242 }
243 break;
244 case CTokStateExpSign:
245 switch (*c) {
246 case '+':
247 case '-':
248 ctok->state = CTokStateFloatExpFirst;
249 buf_append_char(&ctok->buf, *c);
250 break;
251 case DIGIT:
252 ctok->state = CTokStateFloatExp;
253 buf_append_char(&ctok->buf, *c);
254 break;
255 default:
256 return mark_error(ctok);
257 }
258 break;
259 case CTokStateFloatExpFirst:
260 switch (*c) {
261 case DIGIT:
262 buf_append_char(&ctok->buf, *c);
263 ctok->state = CTokStateFloatExp;
264 break;
265 default:
266 return mark_error(ctok);
267 }
268 break;
269 case CTokStateFloatExp:
270 switch (*c) {
271 case DIGIT:
272 buf_append_char(&ctok->buf, *c);
273 break;
274 case 'f':
275 case 'F':
276 case 'l':
277 case 'L':
278 end_float(ctok);
279 break;
280 default:
281 c -= 1;
282 end_float(ctok);
283 continue;
284 }
285 break;
286 case CTokStateDecimal:
287 switch (*c) {
288 case DIGIT:
289 buf_append_char(&ctok->buf, *c);
290
291 // TODO @mul_with_overflow
292 ctok->cur_tok->data.num_lit_int *= 10;
293 // TODO @add_with_overflow
294 ctok->cur_tok->data.num_lit_int += *c - '0';
295 break;
296 case '\'':
297 break;
298 case 'u':
299 case 'U':
300 ctok->unsigned_suffix = true;
301 ctok->state = CTokStateIntSuffix;
302 break;
303 case 'l':
304 case 'L':
305 ctok->long_suffix = true;
306 ctok->state = CTokStateIntSuffixLong;
307 break;
308 case '.':
309 buf_append_char(&ctok->buf, '.');
310 ctok->cur_tok->id = CTokIdNumLitFloat;
311 ctok->state = CTokStateFloat;
312 break;
313 default:
314 c -= 1;
315 end_token(ctok);
316 ctok->state = CTokStateStart;
317 continue;
318 }
319 break;
320 case CTokStateIntSuffix:
321 switch (*c) {
322 case 'l':
323 case 'L':
324 if (ctok->long_suffix) {
325 return mark_error(ctok);
326 }
327 ctok->long_suffix = true;
328 ctok->state = CTokStateIntSuffixLong;
329 break;
330 case 'u':
331 case 'U':
332 if (ctok->unsigned_suffix) {
333 return mark_error(ctok);
334 }
335 ctok->unsigned_suffix = true;
336 break;
337 default:
338 c -= 1;
339 end_token(ctok);
340 ctok->state = CTokStateStart;
341 continue;
342 }
343 break;
344 case CTokStateIntSuffixLong:
345 switch (*c) {
346 case 'l':
347 case 'L':
348 ctok->state = CTokStateIntSuffix;
349 break;
350 case 'u':
351 case 'U':
352 if (ctok->unsigned_suffix) {
353 return mark_error(ctok);
354 }
355 ctok->unsigned_suffix = true;
356 break;
357 default:
358 c -= 1;
359 end_token(ctok);
360 ctok->state = CTokStateStart;
361 continue;
362 }
363 break;
364 case CTokStateGotZero:
365 switch (*c) {
366 case 'x':
367 case 'X':
368 ctok->state = CTokStateHex;
369 break;
370 case '.':
371 ctok->state = CTokStateFloat;
372 ctok->cur_tok->id = CTokIdNumLitFloat;
373 buf_append_char(&ctok->buf, '.');
374 break;
375 default:
376 c -= 1;
377 ctok->state = CTokStateOctal;
378 continue;
379 }
380 break;
381 case CTokStateOctal:
382 switch (*c) {
383 case '0':
384 case '1':
385 case '2':
386 case '3':
387 case '4':
388 case '5':
389 case '6':
390 case '7':
391 // TODO @mul_with_overflow
392 ctok->cur_tok->data.num_lit_int *= 8;
393 // TODO @add_with_overflow
394 ctok->cur_tok->data.num_lit_int += *c - '0';
395 break;
396 case '8':
397 case '9':
398 return mark_error(ctok);
399 case '\'':
400 break;
401 default:
402 c -= 1;
403 end_token(ctok);
404 ctok->state = CTokStateStart;
405 continue;
406 }
407 break;
408 case CTokStateHex:
409 switch (*c) {
410 case '0':
411 hex_digit(ctok, 0);
412 break;
413 case '1':
414 hex_digit(ctok, 1);
415 break;
416 case '2':
417 hex_digit(ctok, 2);
418 break;
419 case '3':
420 hex_digit(ctok, 3);
421 break;
422 case '4':
423 hex_digit(ctok, 4);
424 break;
425 case '5':
426 hex_digit(ctok, 5);
427 break;
428 case '6':
429 hex_digit(ctok, 6);
430 break;
431 case '7':
432 hex_digit(ctok, 7);
433 break;
434 case '8':
435 hex_digit(ctok, 8);
436 break;
437 case '9':
438 hex_digit(ctok, 9);
439 break;
440 case 'a':
441 case 'A':
442 hex_digit(ctok, 10);
443 break;
444 case 'b':
445 case 'B':
446 hex_digit(ctok, 11);
447 break;
448 case 'c':
449 case 'C':
450 hex_digit(ctok, 12);
451 break;
452 case 'd':
453 case 'D':
454 hex_digit(ctok, 13);
455 break;
456 case 'e':
457 case 'E':
458 hex_digit(ctok, 14);
459 break;
460 case 'f':
461 case 'F':
462 hex_digit(ctok, 15);
463 break;
464 case 'p':
465 case 'P':
466 ctok->cur_tok->id = CTokIdNumLitFloat;
467 ctok->state = CTokStateExpSign;
468 break;
469 default:
470 c -= 1;
471 end_token(ctok);
472 ctok->state = CTokStateStart;
473 continue;
474 }
475 break;
476 case CTokStateIdentifier:
477 switch (*c) {
478 case IDENT:
479 buf_append_char(&ctok->cur_tok->data.symbol, *c);
480 break;
481 default:
482 c -= 1;
483 end_token(ctok);
484 ctok->state = CTokStateStart;
485 continue;
486 }
487 break;
488 case CTokStateString:
489 switch (*c) {
490 case '\\':
491 ctok->state = CTokStateCharEscape;
492 break;
493 case '\"':
494 end_token(ctok);
495 ctok->state = CTokStateStart;
496 break;
497 default:
498 buf_append_char(&ctok->cur_tok->data.str_lit, *c);
499 }
500 break;
501 case CTokStateExpectChar:
502 switch (*c) {
503 case '\\':
504 ctok->state = CTokStateCharEscape;
505 break;
506 case '\'':
507 return mark_error(ctok);
508 default:
509 ctok->cur_tok->data.char_lit = *c;
510 ctok->state = CTokStateExpectEndQuot;
511 }
512 break;
513 case CTokStateCharEscape:
514 switch (*c) {
515 case '\'':
516 case '"':
517 case '?':
518 case '\\':
519 add_char(ctok, *c);
520 break;
521 case 'a':
522 add_char(ctok, '\a');
523 break;
524 case 'b':
525 add_char(ctok, '\b');
526 break;
527 case 'f':
528 add_char(ctok, '\f');
529 break;
530 case 'n':
531 add_char(ctok, '\n');
532 break;
533 case 'r':
534 add_char(ctok, '\r');
535 break;
536 case 't':
537 add_char(ctok, '\t');
538 break;
539 case 'v':
540 add_char(ctok, '\v');
541 break;
542 case DIGIT:
543 zig_panic("TODO octal");
544 break;
545 case 'x':
546 zig_panic("TODO hex");
547 break;
548 case 'u':
549 zig_panic("TODO unicode");
550 break;
551 case 'U':
552 zig_panic("TODO Unicode");
553 break;
554 default:
555 return mark_error(ctok);
556 }
557 break;
558 case CTokStateExpectEndQuot:
559 switch (*c) {
560 case '\'':
561 end_token(ctok);
562 ctok->state = CTokStateStart;
563 break;
564 default:
565 return mark_error(ctok);
566 }
567 break;
568 case CTokStateOpenComment:
569 switch (*c) {
570 case '/':
571 ctok->state = CTokStateLineComment;
572 break;
573 case '*':
574 ctok->state = CTokStateComment;
575 break;
576 default:
577 return mark_error(ctok);
578 }
579 break;
580 case CTokStateLineComment:
581 if (*c == '\n') {
582 ctok->state = CTokStateStart;
583 goto found_end_of_macro;
584 }
585 break;
586 case CTokStateComment:
587 switch (*c) {
588 case '*':
589 ctok->state = CTokStateCommentStar;
590 break;
591 default:
592 break;
593 }
594 break;
595 case CTokStateCommentStar:
596 switch (*c) {
597 case '/':
598 ctok->state = CTokStateStart;
599 break;
600 case '*':
601 break;
602 default:
603 ctok->state = CTokStateComment;
604 break;
605 }
606 break;
607 case CTokStateBackslash:
608 switch (*c) {
609 case '\n':
610 ctok->state = CTokStateStart;
611 break;
612 default:
613 return mark_error(ctok);
614 }
615 break;
616 }
617 }
618found_end_of_macro:
619
620 switch (ctok->state) {
621 case CTokStateStart:
622 break;
623 case CTokStateIdentifier:
624 case CTokStateDecimal:
625 case CTokStateHex:
626 case CTokStateOctal:
627 case CTokStateGotZero:
628 case CTokStateIntSuffix:
629 case CTokStateIntSuffixLong:
630 end_token(ctok);
631 break;
632 case CTokStateFloat:
633 case CTokStateFloatExp:
634 end_float(ctok);
635 break;
636 case CTokStateExpectChar:
637 case CTokStateExpectEndQuot:
638 case CTokStateOpenComment:
639 case CTokStateLineComment:
640 case CTokStateComment:
641 case CTokStateCommentStar:
642 case CTokStateCharEscape:
643 case CTokStateBackslash:
644 case CTokStateString:
645 case CTokStateExpSign:
646 case CTokStateFloatExpFirst:
647 return mark_error(ctok);
648 }
649
650 assert(ctok->cur_tok == nullptr);
651}
src/c_tokenizer.hpp created+70
......@@ -0,0 +1,70 @@
1/*
2 * Copyright (c) 2016 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8
9#ifndef ZIG_C_TOKENIZER_HPP
10#define ZIG_C_TOKENIZER_HPP
11
12#include "buffer.hpp"
13
14enum CTokId {
15 CTokIdCharLit,
16 CTokIdStrLit,
17 CTokIdNumLitInt,
18 CTokIdNumLitFloat,
19 CTokIdSymbol,
20 CTokIdMinus,
21};
22
23struct CTok {
24 enum CTokId id;
25 union {
26 uint8_t char_lit;
27 Buf str_lit;
28 uint64_t num_lit_int;
29 double num_lit_float;
30 Buf symbol;
31 } data;
32};
33
34enum CTokState {
35 CTokStateStart,
36 CTokStateExpectChar,
37 CTokStateCharEscape,
38 CTokStateExpectEndQuot,
39 CTokStateOpenComment,
40 CTokStateLineComment,
41 CTokStateComment,
42 CTokStateCommentStar,
43 CTokStateBackslash,
44 CTokStateString,
45 CTokStateIdentifier,
46 CTokStateDecimal,
47 CTokStateOctal,
48 CTokStateGotZero,
49 CTokStateHex,
50 CTokStateIntSuffix,
51 CTokStateIntSuffixLong,
52 CTokStateFloat,
53 CTokStateExpSign,
54 CTokStateFloatExp,
55 CTokStateFloatExpFirst,
56};
57
58struct CTokenize {
59 ZigList<CTok> tokens;
60 CTokState state;
61 bool error;
62 CTok *cur_tok;
63 Buf buf;
64 bool unsigned_suffix;
65 bool long_suffix;
66};
67
68void tokenize_c_macro(CTokenize *ctok, const uint8_t *c);
69
70#endif
src/parseh.cpp+78-204
......@@ -12,6 +12,7 @@
1212#include "parser.hpp"
1313#include "all_types.hpp"
1414#include "tokenizer.hpp"
15#include "c_tokenizer.hpp"
1516#include "analyze.hpp"
1617
1718#include <clang/Frontend/ASTUnit.h>
......@@ -176,6 +177,19 @@ static AstNode *create_str_lit_node(Context *c, Buf *buf) {
176177 return node;
177178}
178179
180static AstNode *create_num_lit_float(Context *c, double x) {
181 AstNode *node = create_node(c, NodeTypeNumberLiteral);
182 node->data.number_literal.kind = NumLitFloat;
183 node->data.number_literal.data.x_float = x;
184 return node;
185}
186
187static AstNode *create_num_lit_float_negative(Context *c, double x, bool negative) {
188 AstNode *num_lit_node = create_num_lit_float(c, x);
189 if (!negative) return num_lit_node;
190 return create_prefix_node(c, PrefixOpNegation, num_lit_node);
191}
192
179193static AstNode *create_num_lit_unsigned(Context *c, uint64_t x) {
180194 AstNode *node = create_node(c, NodeTypeNumberLiteral);
181195 node->data.number_literal.kind = NumLitUInt;
......@@ -183,6 +197,12 @@ static AstNode *create_num_lit_unsigned(Context *c, uint64_t x) {
183197 return node;
184198}
185199
200static AstNode *create_num_lit_unsigned_negative(Context *c, uint64_t x, bool negative) {
201 AstNode *num_lit_node = create_num_lit_unsigned(c, x);
202 if (!negative) return num_lit_node;
203 return create_prefix_node(c, PrefixOpNegation, num_lit_node);
204}
205
186206static AstNode *create_num_lit_signed(Context *c, int64_t x) {
187207 if (x >= 0) {
188208 return create_num_lit_unsigned(c, x);
......@@ -1244,209 +1264,70 @@ static void render_macros(Context *c) {
12441264 }
12451265}
12461266
1247static int parse_c_char_lit(Buf *value, uint8_t *out_c) {
1248 enum State {
1249 StateExpectStartQuot,
1250 StateExpectChar,
1251 StateExpectEndQuot,
1252 StateExpectEnd,
1253 };
1254 State state = StateExpectStartQuot;
1255 for (int i = 0; i < buf_len(value); i += 1) {
1256 uint8_t c = buf_ptr(value)[i];
1257 switch (state) {
1258 case StateExpectStartQuot:
1259 switch (c) {
1260 case '\'':
1261 state = StateExpectChar;
1262 break;
1263 default:
1264 return -1;
1265 }
1266 break;
1267 case StateExpectChar:
1268 switch (c) {
1269 case '\\':
1270 case '\'':
1271 return -1;
1272 default:
1273 *out_c = c;
1274 state = StateExpectEndQuot;
1275 }
1276 break;
1277 case StateExpectEndQuot:
1278 switch (c) {
1279 case '\'':
1280 state = StateExpectEnd;
1281 break;
1282 default:
1283 return -1;
1284 }
1285 break;
1286 case StateExpectEnd:
1287 return -1;
1288 }
1289 }
1290 return (state == StateExpectEnd) ? 0 : -1;
1291}
1292
1293static int parse_c_num_lit_unsigned(Buf *buf, uint64_t *out_val) {
1294 char *temp;
1295 *out_val = strtoull(buf_ptr(buf), &temp, 0);
1296
1297 if (temp == buf_ptr(buf) || *temp != 0 || *out_val == ULLONG_MAX) {
1298 return -1;
1299 }
1300
1301 return 0;
1302}
1303
1304static bool is_simple_symbol(Buf *buf) {
1305 bool first = true;
1306 for (int i = 0; i < buf_len(buf); i += 1) {
1307 uint8_t c = buf_ptr(buf)[i];
1308 bool valid_alpha = (c >= 'a' && c <= 'z') ||
1309 (c >= 'A' && c <= 'Z') || c == '_';
1310 bool valid_digit = (c >= '0' && c <= '9');
1311
1312 bool ok = (valid_alpha || (!first && valid_digit));
1313 first = false;
1314
1315 if (!ok) {
1316 return false;
1317 }
1318 }
1319 return true;
1320}
1321
1322enum ParseCStrState {
1323 ParseCStrStateExpectQuot,
1324 ParseCStrStateNormal,
1325 ParseCStrStateEscape,
1326};
1327
1328static int parse_c_str_lit(Buf *buf, Buf *out_str) {
1329 ParseCStrState state = ParseCStrStateExpectQuot;
1330 buf_resize(out_str, 0);
1331
1332 for (int i = 0; i < buf_len(buf); i += 1) {
1333 uint8_t c = buf_ptr(buf)[i];
1334 switch (state) {
1335 case ParseCStrStateExpectQuot:
1336 if (c == '"') {
1337 state = ParseCStrStateNormal;
1338 } else {
1339 return -1;
1340 }
1341 break;
1342 case ParseCStrStateNormal:
1343 switch (c) {
1344 case '\\':
1345 state = ParseCStrStateEscape;
1346 break;
1347 case '\n':
1348 return -1;
1349 case '"':
1350 return 0;
1351 default:
1352 buf_append_char(out_str, c);
1353 }
1354 break;
1355 case ParseCStrStateEscape:
1356 switch (c) {
1357 case '\'':
1358 buf_append_char(out_str, '\'');
1359 state = ParseCStrStateNormal;
1360 break;
1361 case '"':
1362 buf_append_char(out_str, '"');
1363 state = ParseCStrStateNormal;
1364 break;
1365 case '?':
1366 buf_append_char(out_str, '\?');
1367 state = ParseCStrStateNormal;
1368 break;
1369 case '\\':
1370 buf_append_char(out_str, '\\');
1371 state = ParseCStrStateNormal;
1372 break;
1373 case 'a':
1374 buf_append_char(out_str, '\a');
1375 state = ParseCStrStateNormal;
1376 break;
1377 case 'b':
1378 buf_append_char(out_str, '\b');
1379 state = ParseCStrStateNormal;
1380 break;
1381 case 'f':
1382 buf_append_char(out_str, '\f');
1383 state = ParseCStrStateNormal;
1384 break;
1385 case 'n':
1386 buf_append_char(out_str, '\n');
1387 state = ParseCStrStateNormal;
1388 break;
1389 case 'r':
1390 buf_append_char(out_str, '\r');
1391 state = ParseCStrStateNormal;
1392 break;
1393 case 't':
1394 buf_append_char(out_str, '\t');
1395 state = ParseCStrStateNormal;
1396 break;
1397 case 'v':
1398 buf_append_char(out_str, '\v');
1399 state = ParseCStrStateNormal;
1400 break;
1401 default:
1402 // TODO octal escape sequence, hexadecimal escape sequence, and
1403 // universal character name
1404 return -1;
1405 }
1406 break;
1407 }
1408 }
1409
1410 return -1;
1411}
1412
1413static void process_macro(Context *c, Buf *name, Buf *value) {
1414 //fprintf(stderr, "macro '%s' = '%s'\n", buf_ptr(name), buf_ptr(value));
1267static void process_macro(Context *c, CTokenize *ctok, Buf *name, const char *char_ptr) {
14151268 if (is_zig_keyword(name)) {
14161269 return;
14171270 }
14181271
1419 // maybe it's a character literal
1420 uint8_t ch;
1421 if (!parse_c_char_lit(value, &ch)) {
1422 AstNode *var_node = create_var_decl_node(c, buf_ptr(name), create_char_lit_node(c, ch));
1423 c->macro_table.put(name, var_node);
1424 return;
1425 }
1426 // maybe it's a string literal
1427 Buf str_lit = BUF_INIT;
1428 if (!parse_c_str_lit(value, &str_lit)) {
1429 AstNode *var_node = create_var_decl_node(c, buf_ptr(name), create_str_lit_node(c, &str_lit));
1430 c->macro_table.put(name, var_node);
1431 return;
1432 }
1272 tokenize_c_macro(ctok, (const uint8_t *)char_ptr);
14331273
1434 // maybe it's an unsigned integer
1435 uint64_t uint;
1436 if (!parse_c_num_lit_unsigned(value, &uint)) {
1437 AstNode *var_node = create_var_decl_node(c, buf_ptr(name), create_num_lit_unsigned(c, uint));
1438 c->macro_table.put(name, var_node);
1274 if (ctok->error) {
14391275 return;
14401276 }
14411277
1442 // maybe it's a symbol
1443 if (is_simple_symbol(value)) {
1444 // if it equals itself, ignore. for example, from stdio.h:
1445 // #define stdin stdin
1446 if (buf_eql_buf(name, value)) {
1447 return;
1278 bool negate = false;
1279 for (int i = 0; i < ctok->tokens.length; i += 1) {
1280 bool is_first = (i == 0);
1281 bool is_last = (i == ctok->tokens.length - 1);
1282 CTok *tok = &ctok->tokens.at(i);
1283 switch (tok->id) {
1284 case CTokIdCharLit:
1285 if (is_last && is_first) {
1286 AstNode *var_node = create_var_decl_node(c, buf_ptr(name),
1287 create_char_lit_node(c, tok->data.char_lit));
1288 c->macro_table.put(name, var_node);
1289 }
1290 return;
1291 case CTokIdStrLit:
1292 if (is_last && is_first) {
1293 AstNode *var_node = create_var_decl_node(c, buf_ptr(name),
1294 create_str_lit_node(c, &tok->data.str_lit));
1295 c->macro_table.put(name, var_node);
1296 }
1297 return;
1298 case CTokIdNumLitInt:
1299 if (is_last) {
1300 AstNode *var_node = create_var_decl_node(c, buf_ptr(name),
1301 create_num_lit_unsigned_negative(c, tok->data.num_lit_int, negate));
1302 c->macro_table.put(name, var_node);
1303 }
1304 return;
1305 case CTokIdNumLitFloat:
1306 if (is_last) {
1307 AstNode *var_node = create_var_decl_node(c, buf_ptr(name),
1308 create_num_lit_float_negative(c, tok->data.num_lit_float, negate));
1309 c->macro_table.put(name, var_node);
1310 }
1311 return;
1312 case CTokIdSymbol:
1313 if (is_last && is_first) {
1314 // if it equals itself, ignore. for example, from stdio.h:
1315 // #define stdin stdin
1316 Buf *symbol_name = buf_create_from_buf(&tok->data.symbol);
1317 if (buf_eql_buf(name, symbol_name)) {
1318 return;
1319 }
1320 c->macro_symbols.append({name, symbol_name});
1321 return;
1322 }
1323 case CTokIdMinus:
1324 if (is_first) {
1325 negate = true;
1326 break;
1327 } else {
1328 return;
1329 }
14481330 }
1449 c->macro_symbols.append({name, value});
14501331 }
14511332}
14521333
......@@ -1473,6 +1354,8 @@ static void process_symbol_macros(Context *c) {
14731354}
14741355
14751356static void process_preprocessor_entities(Context *c, ASTUnit &unit) {
1357 CTokenize ctok = {{0}};
1358
14761359 for (PreprocessedEntity *entity : unit.getLocalPreprocessingEntities()) {
14771360 switch (entity->getKind()) {
14781361 case PreprocessedEntity::InvalidKind:
......@@ -1494,16 +1377,7 @@ static void process_preprocessor_entities(Context *c, ASTUnit &unit) {
14941377 }
14951378
14961379 const char *end_c = c->source_manager->getCharacterData(end_loc);
1497 Buf *value = buf_alloc();
1498 while (*end_c && *end_c != '\n') {
1499 buf_append_char(value, *end_c);
1500 if (end_c[0] == '\\' && end_c[1] == '\n') {
1501 end_c += 2;
1502 } else {
1503 end_c += 1;
1504 }
1505 }
1506 process_macro(c, buf_create_from_str(name), value);
1380 process_macro(c, &ctok, buf_create_from_str(name), end_c);
15071381 }
15081382 }
15091383 }
test/run_tests.cpp+4
......@@ -1390,6 +1390,10 @@ extern void (*fn_ptr)(void);
13901390 add_parseh_case("__cdecl doesn't mess up function pointers", R"SOURCE(
13911391void foo(void (__cdecl *fn_ptr)(void));
13921392 )SOURCE", 1, "pub extern fn foo(fn_ptr: ?extern fn());");
1393
1394 add_parseh_case("comment after integer literal", R"SOURCE(
1395#define SDL_INIT_VIDEO 0x00000020 /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
1396 )SOURCE", 1, "pub const SDL_INIT_VIDEO = 32;");
13931397}
13941398
13951399static void run_self_hosted_test(void) {