authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-02-05 14:29:39-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-02-05 14:29:39-05:00
log40b1fecace87abc2e508bfafea955b3d08266301
tree5f13e57e47472a33692cf302303636ff63a3f8cb
parentb022db16ece48f2cdec47c5716601fec2fc07ef4
parentf196ddd251d80fac74685d822ec26a646c7890fa
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4363 from Vexu/trans-std-c

Use std.c.tokenizer in translate-c

4 files changed, 348 insertions(+), 1078 deletions(-)

lib/std/c/tokenizer.zig+4-1
...@@ -776,12 +776,14 @@ pub const Tokenizer = struct {...@@ -776,12 +776,14 @@ pub const Tokenizer = struct {
776 }776 }
777 },777 },
778 else => {778 else => {
779 self.index -= 1;
779 state = if (string) .StringLiteral else .CharLiteral;780 state = if (string) .StringLiteral else .CharLiteral;
780 },781 },
781 },782 },
782 .HexEscape => switch (c) {783 .HexEscape => switch (c) {
783 '0'...'9', 'a'...'f', 'A'...'F' => {},784 '0'...'9', 'a'...'f', 'A'...'F' => {},
784 else => {785 else => {
786 self.index -= 1;
785 state = if (string) .StringLiteral else .CharLiteral;787 state = if (string) .StringLiteral else .CharLiteral;
786 },788 },
787 },789 },
...@@ -797,6 +799,7 @@ pub const Tokenizer = struct {...@@ -797,6 +799,7 @@ pub const Tokenizer = struct {
797 result.id = .Invalid;799 result.id = .Invalid;
798 break;800 break;
799 }801 }
802 self.index -= 1;
800 state = if (string) .StringLiteral else .CharLiteral;803 state = if (string) .StringLiteral else .CharLiteral;
801 },804 },
802 },805 },
...@@ -1046,7 +1049,6 @@ pub const Tokenizer = struct {...@@ -1046,7 +1049,6 @@ pub const Tokenizer = struct {
1046 .LineComment => switch (c) {1049 .LineComment => switch (c) {
1047 '\n' => {1050 '\n' => {
1048 result.id = .LineComment;1051 result.id = .LineComment;
1049 self.index += 1;
1050 break;1052 break;
1051 },1053 },
1052 else => {},1054 else => {},
...@@ -1217,6 +1219,7 @@ pub const Tokenizer = struct {...@@ -1217,6 +1219,7 @@ pub const Tokenizer = struct {
1217 result.id = .Invalid;1219 result.id = .Invalid;
1218 break;1220 break;
1219 }1221 }
1222 self.index -= 1;
1220 state = .FloatSuffix;1223 state = .FloatSuffix;
1221 },1224 },
1222 },1225 },
src-self-hosted/c_tokenizer.zig deleted-977
...@@ -1,977 +0,0 @@
1const std = @import("std");
2const expect = std.testing.expect;
3const ZigClangSourceLocation = @import("clang.zig").ZigClangSourceLocation;
4const Context = @import("translate_c.zig").Context;
5const failDecl = @import("translate_c.zig").failDecl;
6
7pub const TokenList = std.SegmentedList(CToken, 32);
8
9pub const CToken = struct {
10 id: Id,
11 bytes: []const u8 = "",
12 num_lit_suffix: NumLitSuffix = .None,
13
14 pub const Id = enum {
15 CharLit,
16 StrLit,
17 NumLitInt,
18 NumLitFloat,
19 Identifier,
20 Plus,
21 Minus,
22 Slash,
23 LParen,
24 RParen,
25 Eof,
26 Dot,
27 Asterisk, // *
28 Ampersand, // &
29 And, // &&
30 Assign, // =
31 Or, // ||
32 Bang, // !
33 Tilde, // ~
34 Shl, // <<
35 Shr, // >>
36 Lt, // <
37 Lte, // <=
38 Gt, // >
39 Gte, // >=
40 Eq, // ==
41 Ne, // !=
42 Increment, // ++
43 Decrement, // --
44 Comma,
45 Fn,
46 Arrow, // ->
47 LBrace,
48 RBrace,
49 Pipe,
50 QuestionMark,
51 Colon,
52 };
53
54 pub const NumLitSuffix = enum {
55 None,
56 F,
57 L,
58 U,
59 LU,
60 LL,
61 LLU,
62 };
63};
64
65pub fn tokenizeCMacro(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, tl: *TokenList, chars: [*:0]const u8) !void {
66 var index: usize = 0;
67 var first = true;
68 while (true) {
69 const tok = try next(ctx, loc, name, chars, &index);
70 if (tok.id == .StrLit or tok.id == .CharLit)
71 try tl.push(try zigifyEscapeSequences(ctx, loc, name, tl.allocator, tok))
72 else
73 try tl.push(tok);
74 if (tok.id == .Eof)
75 return;
76 if (first) {
77 // distinguish NAME (EXPR) from NAME(ARGS)
78 first = false;
79 if (chars[index] == '(') {
80 try tl.push(.{
81 .id = .Fn,
82 .bytes = "",
83 });
84 }
85 }
86 }
87}
88
89fn zigifyEscapeSequences(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, allocator: *std.mem.Allocator, tok: CToken) !CToken {
90 for (tok.bytes) |c| {
91 if (c == '\\') {
92 break;
93 }
94 } else return tok;
95 var bytes = try allocator.alloc(u8, tok.bytes.len * 2);
96 var state: enum {
97 Start,
98 Escape,
99 Hex,
100 Octal,
101 } = .Start;
102 var i: usize = 0;
103 var count: u8 = 0;
104 var num: u8 = 0;
105 for (tok.bytes) |c| {
106 switch (state) {
107 .Escape => {
108 switch (c) {
109 'n', 'r', 't', '\\', '\'', '\"' => {
110 bytes[i] = c;
111 },
112 '0'...'7' => {
113 count += 1;
114 num += c - '0';
115 state = .Octal;
116 bytes[i] = 'x';
117 },
118 'x' => {
119 state = .Hex;
120 bytes[i] = 'x';
121 },
122 'a' => {
123 bytes[i] = 'x';
124 i += 1;
125 bytes[i] = '0';
126 i += 1;
127 bytes[i] = '7';
128 },
129 'b' => {
130 bytes[i] = 'x';
131 i += 1;
132 bytes[i] = '0';
133 i += 1;
134 bytes[i] = '8';
135 },
136 'f' => {
137 bytes[i] = 'x';
138 i += 1;
139 bytes[i] = '0';
140 i += 1;
141 bytes[i] = 'C';
142 },
143 'v' => {
144 bytes[i] = 'x';
145 i += 1;
146 bytes[i] = '0';
147 i += 1;
148 bytes[i] = 'B';
149 },
150 '?' => {
151 i -= 1;
152 bytes[i] = '?';
153 },
154 'u', 'U' => {
155 try failDecl(ctx, loc, name, "macro tokenizing failed: TODO unicode escape sequences", .{});
156 return error.TokenizingFailed;
157 },
158 else => {
159 try failDecl(ctx, loc, name, "macro tokenizing failed: unknown escape sequence", .{});
160 return error.TokenizingFailed;
161 },
162 }
163 i += 1;
164 if (state == .Escape)
165 state = .Start;
166 },
167 .Start => {
168 if (c == '\\') {
169 state = .Escape;
170 }
171 bytes[i] = c;
172 i += 1;
173 },
174 .Hex => {
175 switch (c) {
176 '0'...'9' => {
177 num = std.math.mul(u8, num, 16) catch {
178 try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
179 return error.TokenizingFailed;
180 };
181 num += c - '0';
182 },
183 'a'...'f' => {
184 num = std.math.mul(u8, num, 16) catch {
185 try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
186 return error.TokenizingFailed;
187 };
188 num += c - 'a' + 10;
189 },
190 'A'...'F' => {
191 num = std.math.mul(u8, num, 16) catch {
192 try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
193 return error.TokenizingFailed;
194 };
195 num += c - 'A' + 10;
196 },
197 else => {
198 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
199 num = 0;
200 if (c == '\\')
201 state = .Escape
202 else
203 state = .Start;
204 bytes[i] = c;
205 i += 1;
206 },
207 }
208 },
209 .Octal => {
210 const accept_digit = switch (c) {
211 // The maximum length of a octal literal is 3 digits
212 '0'...'7' => count < 3,
213 else => false,
214 };
215
216 if (accept_digit) {
217 count += 1;
218 num = std.math.mul(u8, num, 8) catch {
219 try failDecl(ctx, loc, name, "macro tokenizing failed: octal literal overflowed", .{});
220 return error.TokenizingFailed;
221 };
222 num += c - '0';
223 } else {
224 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
225 num = 0;
226 count = 0;
227 if (c == '\\')
228 state = .Escape
229 else
230 state = .Start;
231 bytes[i] = c;
232 i += 1;
233 }
234 },
235 }
236 }
237 if (state == .Hex or state == .Octal)
238 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
239 return CToken{
240 .id = tok.id,
241 .bytes = bytes[0..i],
242 };
243}
244
245fn next(ctx: *Context, loc: ZigClangSourceLocation, name: []const u8, chars: [*:0]const u8, i: *usize) !CToken {
246 var state: enum {
247 Start,
248 SawLt,
249 SawGt,
250 SawPlus,
251 SawMinus,
252 SawAmpersand,
253 SawPipe,
254 SawBang,
255 SawEq,
256 CharLit,
257 OpenComment,
258 Comment,
259 CommentStar,
260 Backslash,
261 String,
262 Identifier,
263 Decimal,
264 Octal,
265 SawZero,
266 Hex,
267 Bin,
268 Float,
269 ExpSign,
270 FloatExp,
271 FloatExpFirst,
272 NumLitIntSuffixU,
273 NumLitIntSuffixL,
274 NumLitIntSuffixLL,
275 NumLitIntSuffixUL,
276 Done,
277 } = .Start;
278
279 var result = CToken{
280 .bytes = "",
281 .id = .Eof,
282 };
283 var begin_index: usize = 0;
284 var digits: u8 = 0;
285 var pre_escape = state;
286
287 while (true) {
288 const c = chars[i.*];
289 if (c == 0) {
290 switch (state) {
291 .Identifier,
292 .Decimal,
293 .Hex,
294 .Bin,
295 .Octal,
296 .SawZero,
297 .Float,
298 .FloatExp,
299 => {
300 result.bytes = chars[begin_index..i.*];
301 return result;
302 },
303 .Start,
304 .SawMinus,
305 .Done,
306 .NumLitIntSuffixU,
307 .NumLitIntSuffixL,
308 .NumLitIntSuffixUL,
309 .NumLitIntSuffixLL,
310 .SawLt,
311 .SawGt,
312 .SawPlus,
313 .SawAmpersand,
314 .SawPipe,
315 .SawBang,
316 .SawEq,
317 => {
318 return result;
319 },
320 .CharLit,
321 .OpenComment,
322 .Comment,
323 .CommentStar,
324 .Backslash,
325 .String,
326 .ExpSign,
327 .FloatExpFirst,
328 => {
329 try failDecl(ctx, loc, name, "macro tokenizing failed: unexpected EOF", .{});
330 return error.TokenizingFailed;
331 },
332 }
333 }
334 switch (state) {
335 .Start => {
336 switch (c) {
337 ' ', '\t', '\x0B', '\x0C' => {},
338 '\'' => {
339 state = .CharLit;
340 result.id = .CharLit;
341 begin_index = i.*;
342 },
343 '\"' => {
344 state = .String;
345 result.id = .StrLit;
346 begin_index = i.*;
347 },
348 '/' => {
349 state = .OpenComment;
350 },
351 '\\' => {
352 state = .Backslash;
353 },
354 '\n', '\r' => {
355 return result;
356 },
357 'a'...'z', 'A'...'Z', '_' => {
358 state = .Identifier;
359 result.id = .Identifier;
360 begin_index = i.*;
361 },
362 '1'...'9' => {
363 state = .Decimal;
364 result.id = .NumLitInt;
365 begin_index = i.*;
366 },
367 '0' => {
368 state = .SawZero;
369 result.id = .NumLitInt;
370 begin_index = i.*;
371 },
372 '.' => {
373 result.id = .Dot;
374 state = .Done;
375 },
376 '<' => {
377 result.id = .Lt;
378 state = .SawLt;
379 },
380 '>' => {
381 result.id = .Gt;
382 state = .SawGt;
383 },
384 '(' => {
385 result.id = .LParen;
386 state = .Done;
387 },
388 ')' => {
389 result.id = .RParen;
390 state = .Done;
391 },
392 '*' => {
393 result.id = .Asterisk;
394 state = .Done;
395 },
396 '+' => {
397 result.id = .Plus;
398 state = .SawPlus;
399 },
400 '-' => {
401 result.id = .Minus;
402 state = .SawMinus;
403 },
404 '!' => {
405 result.id = .Bang;
406 state = .SawBang;
407 },
408 '~' => {
409 result.id = .Tilde;
410 state = .Done;
411 },
412 '=' => {
413 result.id = .Assign;
414 state = .SawEq;
415 },
416 ',' => {
417 result.id = .Comma;
418 state = .Done;
419 },
420 '[' => {
421 result.id = .LBrace;
422 state = .Done;
423 },
424 ']' => {
425 result.id = .RBrace;
426 state = .Done;
427 },
428 '|' => {
429 result.id = .Pipe;
430 state = .SawPipe;
431 },
432 '&' => {
433 result.id = .Ampersand;
434 state = .SawAmpersand;
435 },
436 '?' => {
437 result.id = .QuestionMark;
438 state = .Done;
439 },
440 ':' => {
441 result.id = .Colon;
442 state = .Done;
443 },
444 else => {
445 try failDecl(ctx, loc, name, "macro tokenizing failed: unexpected character '{c}'", .{c});
446 return error.TokenizingFailed;
447 },
448 }
449 },
450 .Done => return result,
451 .SawMinus => {
452 switch (c) {
453 '>' => {
454 result.id = .Arrow;
455 state = .Done;
456 },
457 '-' => {
458 result.id = .Decrement;
459 state = .Done;
460 },
461 else => return result,
462 }
463 },
464 .SawPlus => {
465 switch (c) {
466 '+' => {
467 result.id = .Increment;
468 state = .Done;
469 },
470 else => return result,
471 }
472 },
473 .SawLt => {
474 switch (c) {
475 '<' => {
476 result.id = .Shl;
477 state = .Done;
478 },
479 '=' => {
480 result.id = .Lte;
481 state = .Done;
482 },
483 else => return result,
484 }
485 },
486 .SawGt => {
487 switch (c) {
488 '>' => {
489 result.id = .Shr;
490 state = .Done;
491 },
492 '=' => {
493 result.id = .Gte;
494 state = .Done;
495 },
496 else => return result,
497 }
498 },
499 .SawPipe => {
500 switch (c) {
501 '|' => {
502 result.id = .Or;
503 state = .Done;
504 },
505 else => return result,
506 }
507 },
508 .SawAmpersand => {
509 switch (c) {
510 '&' => {
511 result.id = .And;
512 state = .Done;
513 },
514 else => return result,
515 }
516 },
517 .SawBang => {
518 switch (c) {
519 '=' => {
520 result.id = .Ne;
521 state = .Done;
522 },
523 else => return result,
524 }
525 },
526 .SawEq => {
527 switch (c) {
528 '=' => {
529 result.id = .Eq;
530 state = .Done;
531 },
532 else => return result,
533 }
534 },
535 .Float => {
536 switch (c) {
537 '.', '0'...'9' => {},
538 'e', 'E' => {
539 state = .ExpSign;
540 },
541 'f',
542 'F',
543 => {
544 result.num_lit_suffix = .F;
545 result.bytes = chars[begin_index..i.*];
546 state = .Done;
547 },
548 'l', 'L' => {
549 result.num_lit_suffix = .L;
550 result.bytes = chars[begin_index..i.*];
551 state = .Done;
552 },
553 else => {
554 result.bytes = chars[begin_index..i.*];
555 return result;
556 },
557 }
558 },
559 .ExpSign => {
560 switch (c) {
561 '+', '-' => {
562 state = .FloatExpFirst;
563 },
564 '0'...'9' => {
565 state = .FloatExp;
566 },
567 else => {
568 try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit or '+' or '-'", .{});
569 return error.TokenizingFailed;
570 },
571 }
572 },
573 .FloatExpFirst => {
574 switch (c) {
575 '0'...'9' => {
576 state = .FloatExp;
577 },
578 else => {
579 try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit", .{});
580 return error.TokenizingFailed;
581 },
582 }
583 },
584 .FloatExp => {
585 switch (c) {
586 '0'...'9' => {},
587 'f', 'F' => {
588 result.num_lit_suffix = .F;
589 result.bytes = chars[begin_index..i.*];
590 state = .Done;
591 },
592 'l', 'L' => {
593 result.num_lit_suffix = .L;
594 result.bytes = chars[begin_index..i.*];
595 state = .Done;
596 },
597 else => {
598 result.bytes = chars[begin_index..i.*];
599 return result;
600 },
601 }
602 },
603 .Decimal => {
604 switch (c) {
605 '0'...'9' => {},
606 '\'' => {},
607 'u', 'U' => {
608 state = .NumLitIntSuffixU;
609 result.num_lit_suffix = .U;
610 result.bytes = chars[begin_index..i.*];
611 },
612 'l', 'L' => {
613 state = .NumLitIntSuffixL;
614 result.num_lit_suffix = .L;
615 result.bytes = chars[begin_index..i.*];
616 },
617 '.' => {
618 result.id = .NumLitFloat;
619 state = .Float;
620 },
621 else => {
622 result.bytes = chars[begin_index..i.*];
623 return result;
624 },
625 }
626 },
627 .SawZero => {
628 switch (c) {
629 'x', 'X' => {
630 state = .Hex;
631 },
632 'b', 'B' => {
633 state = .Bin;
634 },
635 '.' => {
636 state = .Float;
637 result.id = .NumLitFloat;
638 },
639 'u', 'U' => {
640 state = .NumLitIntSuffixU;
641 result.num_lit_suffix = .U;
642 result.bytes = chars[begin_index..i.*];
643 },
644 'l', 'L' => {
645 state = .NumLitIntSuffixL;
646 result.num_lit_suffix = .L;
647 result.bytes = chars[begin_index..i.*];
648 },
649 else => {
650 i.* -= 1;
651 state = .Octal;
652 },
653 }
654 },
655 .Octal => {
656 switch (c) {
657 '0'...'7' => {},
658 '8', '9' => {
659 try failDecl(ctx, loc, name, "macro tokenizing failed: invalid digit '{c}' in octal number", .{c});
660 return error.TokenizingFailed;
661 },
662 'u', 'U' => {
663 state = .NumLitIntSuffixU;
664 result.num_lit_suffix = .U;
665 result.bytes = chars[begin_index..i.*];
666 },
667 'l', 'L' => {
668 state = .NumLitIntSuffixL;
669 result.num_lit_suffix = .L;
670 result.bytes = chars[begin_index..i.*];
671 },
672 else => {
673 result.bytes = chars[begin_index..i.*];
674 return result;
675 },
676 }
677 },
678 .Hex => {
679 switch (c) {
680 '0'...'9', 'a'...'f', 'A'...'F' => {},
681 'u', 'U' => {
682 // marks the number literal as unsigned
683 state = .NumLitIntSuffixU;
684 result.num_lit_suffix = .U;
685 result.bytes = chars[begin_index..i.*];
686 },
687 'l', 'L' => {
688 // marks the number literal as long
689 state = .NumLitIntSuffixL;
690 result.num_lit_suffix = .L;
691 result.bytes = chars[begin_index..i.*];
692 },
693 else => {
694 result.bytes = chars[begin_index..i.*];
695 return result;
696 },
697 }
698 },
699 .Bin => {
700 switch (c) {
701 '0'...'1' => {},
702 '2'...'9' => {
703 try failDecl(ctx, loc, name, "macro tokenizing failed: invalid digit '{c}' in binary number", .{c});
704 return error.TokenizingFailed;
705 },
706 'u', 'U' => {
707 // marks the number literal as unsigned
708 state = .NumLitIntSuffixU;
709 result.num_lit_suffix = .U;
710 result.bytes = chars[begin_index..i.*];
711 },
712 'l', 'L' => {
713 // marks the number literal as long
714 state = .NumLitIntSuffixL;
715 result.num_lit_suffix = .L;
716 result.bytes = chars[begin_index..i.*];
717 },
718 else => {
719 result.bytes = chars[begin_index..i.*];
720 return result;
721 },
722 }
723 },
724 .NumLitIntSuffixU => {
725 switch (c) {
726 'l', 'L' => {
727 result.num_lit_suffix = .LU;
728 state = .NumLitIntSuffixUL;
729 },
730 else => {
731 return result;
732 },
733 }
734 },
735 .NumLitIntSuffixL => {
736 switch (c) {
737 'l', 'L' => {
738 result.num_lit_suffix = .LL;
739 state = .NumLitIntSuffixLL;
740 },
741 'u', 'U' => {
742 result.num_lit_suffix = .LU;
743 state = .Done;
744 },
745 else => {
746 return result;
747 },
748 }
749 },
750 .NumLitIntSuffixLL => {
751 switch (c) {
752 'u', 'U' => {
753 result.num_lit_suffix = .LLU;
754 state = .Done;
755 },
756 else => {
757 return result;
758 },
759 }
760 },
761 .NumLitIntSuffixUL => {
762 switch (c) {
763 'l', 'L' => {
764 result.num_lit_suffix = .LLU;
765 state = .Done;
766 },
767 else => {
768 return result;
769 },
770 }
771 },
772 .Identifier => {
773 switch (c) {
774 '_', 'a'...'z', 'A'...'Z', '0'...'9' => {},
775 else => {
776 result.bytes = chars[begin_index..i.*];
777 return result;
778 },
779 }
780 },
781 .String => {
782 switch (c) {
783 '\"' => {
784 result.bytes = chars[begin_index .. i.* + 1];
785 state = .Done;
786 },
787 else => {},
788 }
789 },
790 .CharLit => {
791 switch (c) {
792 '\'' => {
793 result.bytes = chars[begin_index .. i.* + 1];
794 state = .Done;
795 },
796 else => {},
797 }
798 },
799 .OpenComment => {
800 switch (c) {
801 '/' => {
802 return result;
803 },
804 '*' => {
805 state = .Comment;
806 },
807 else => {
808 result.id = .Slash;
809 state = .Done;
810 },
811 }
812 },
813 .Comment => {
814 switch (c) {
815 '*' => {
816 state = .CommentStar;
817 },
818 else => {},
819 }
820 },
821 .CommentStar => {
822 switch (c) {
823 '/' => {
824 state = .Start;
825 },
826 else => {
827 state = .Comment;
828 },
829 }
830 },
831 .Backslash => {
832 switch (c) {
833 ' ', '\t', '\x0B', '\x0C' => {},
834 '\n', '\r' => {
835 state = .Start;
836 },
837 else => {
838 try failDecl(ctx, loc, name, "macro tokenizing failed: expected whitespace", .{});
839 return error.TokenizingFailed;
840 },
841 }
842 },
843 }
844 i.* += 1;
845 }
846 unreachable;
847}
848
849fn expectTokens(tl: *TokenList, src: [*:0]const u8, expected: []CToken) void {
850 // these can be undefined since they are only used for error reporting
851 tokenizeCMacro(undefined, undefined, undefined, tl, src) catch unreachable;
852 var it = tl.iterator(0);
853 for (expected) |t| {
854 var tok = it.next().?;
855 std.testing.expectEqual(t.id, tok.id);
856 if (t.bytes.len > 0) {
857 //std.debug.warn(" {} = {}\n", .{tok.bytes, t.bytes});
858 std.testing.expectEqualSlices(u8, tok.bytes, t.bytes);
859 }
860 if (t.num_lit_suffix != .None) {
861 std.testing.expectEqual(t.num_lit_suffix, tok.num_lit_suffix);
862 }
863 }
864 std.testing.expect(it.next() == null);
865 tl.shrink(0);
866}
867
868test "tokenize macro" {
869 var tl = TokenList.init(std.testing.allocator);
870 defer tl.deinit();
871
872 expectTokens(&tl, "TEST(0\n", &[_]CToken{
873 .{ .id = .Identifier, .bytes = "TEST" },
874 .{ .id = .Fn },
875 .{ .id = .LParen },
876 .{ .id = .NumLitInt, .bytes = "0" },
877 .{ .id = .Eof },
878 });
879
880 expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{
881 .{ .id = .Identifier, .bytes = "__FLT_MIN_10_EXP__" },
882 .{ .id = .Minus },
883 .{ .id = .NumLitInt, .bytes = "37" },
884 .{ .id = .Eof },
885 });
886
887 expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{
888 .{ .id = .Identifier, .bytes = "__llvm__" },
889 .{ .id = .NumLitInt, .bytes = "1" },
890 .{ .id = .Eof },
891 });
892
893 expectTokens(&tl, "TEST 2", &[_]CToken{
894 .{ .id = .Identifier, .bytes = "TEST" },
895 .{ .id = .NumLitInt, .bytes = "2" },
896 .{ .id = .Eof },
897 });
898
899 expectTokens(&tl, "FOO 0ull", &[_]CToken{
900 .{ .id = .Identifier, .bytes = "FOO" },
901 .{ .id = .NumLitInt, .bytes = "0", .num_lit_suffix = .LLU },
902 .{ .id = .Eof },
903 });
904}
905
906test "tokenize macro ops" {
907 var tl = TokenList.init(std.testing.allocator);
908 defer tl.deinit();
909
910 expectTokens(&tl, "ADD A + B", &[_]CToken{
911 .{ .id = .Identifier, .bytes = "ADD" },
912 .{ .id = .Identifier, .bytes = "A" },
913 .{ .id = .Plus },
914 .{ .id = .Identifier, .bytes = "B" },
915 .{ .id = .Eof },
916 });
917
918 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
919 .{ .id = .Identifier, .bytes = "ADD" },
920 .{ .id = .LParen },
921 .{ .id = .Identifier, .bytes = "A" },
922 .{ .id = .RParen },
923 .{ .id = .Plus },
924 .{ .id = .Identifier, .bytes = "B" },
925 .{ .id = .Eof },
926 });
927
928 expectTokens(&tl, "ADD (A) + B", &[_]CToken{
929 .{ .id = .Identifier, .bytes = "ADD" },
930 .{ .id = .LParen },
931 .{ .id = .Identifier, .bytes = "A" },
932 .{ .id = .RParen },
933 .{ .id = .Plus },
934 .{ .id = .Identifier, .bytes = "B" },
935 .{ .id = .Eof },
936 });
937}
938
939test "escape sequences" {
940 var buf: [1024]u8 = undefined;
941 var alloc = std.heap.FixedBufferAllocator.init(buf[0..]);
942 const a = &alloc.allocator;
943 // these can be undefined since they are only used for error reporting
944 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
945 .id = .StrLit,
946 .bytes = "\\x0077",
947 })).bytes, "\\x77"));
948 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
949 .id = .StrLit,
950 .bytes = "\\24500",
951 })).bytes, "\\xa500"));
952 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
953 .id = .StrLit,
954 .bytes = "\\x0077 abc",
955 })).bytes, "\\x77 abc"));
956 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
957 .id = .StrLit,
958 .bytes = "\\045abc",
959 })).bytes, "\\x25abc"));
960
961 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
962 .id = .CharLit,
963 .bytes = "\\0",
964 })).bytes, "\\x00"));
965 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
966 .id = .CharLit,
967 .bytes = "\\00",
968 })).bytes, "\\x00"));
969 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
970 .id = .CharLit,
971 .bytes = "\\000\\001",
972 })).bytes, "\\x00\\x01"));
973 expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
974 .id = .CharLit,
975 .bytes = "\\000abc",
976 })).bytes, "\\x00abc"));
977}
src-self-hosted/translate_c.zig+320-98
...@@ -6,8 +6,9 @@ const assert = std.debug.assert;...@@ -6,8 +6,9 @@ const assert = std.debug.assert;
6const ast = std.zig.ast;6const ast = std.zig.ast;
7const Token = std.zig.Token;7const Token = std.zig.Token;
8usingnamespace @import("clang.zig");8usingnamespace @import("clang.zig");
9const ctok = @import("c_tokenizer.zig");9const ctok = std.c.tokenizer;
10const CToken = ctok.CToken;10const CToken = std.c.Token;
11const CTokenList = std.c.tokenizer.Source.TokenList;
11const mem = std.mem;12const mem = std.mem;
12const math = std.math;13const math = std.math;
1314
...@@ -4810,6 +4811,15 @@ fn transCreateNodeIdentifier(c: *Context, name: []const u8) !*ast.Node {...@@ -4810,6 +4811,15 @@ fn transCreateNodeIdentifier(c: *Context, name: []const u8) !*ast.Node {
4810 return &identifier.base;4811 return &identifier.base;
4811}4812}
48124813
4814fn transCreateNodeTypeIdentifier(c: *Context, name: []const u8) !*ast.Node {
4815 const token_index = try appendTokenFmt(c, .Identifier, "{}", .{name});
4816 const identifier = try c.a().create(ast.Node.Identifier);
4817 identifier.* = .{
4818 .token = token_index,
4819 };
4820 return &identifier.base;
4821}
4822
4813pub fn freeErrors(errors: []ClangErrMsg) void {4823pub fn freeErrors(errors: []ClangErrMsg) void {
4814 ZigClangErrorMsg_delete(errors.ptr, errors.len);4824 ZigClangErrorMsg_delete(errors.ptr, errors.len);
4815}4825}
...@@ -4818,7 +4828,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {...@@ -4818,7 +4828,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
4818 // TODO if we see #undef, delete it from the table4828 // TODO if we see #undef, delete it from the table
4819 var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit);4829 var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit);
4820 const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit);4830 const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit);
4821 var tok_list = ctok.TokenList.init(c.a());4831 var tok_list = CTokenList.init(c.a());
4822 const scope = c.global_scope;4832 const scope = c.global_scope;
48234833
4824 while (it.I != it_end.I) : (it.I += 1) {4834 while (it.I != it_end.I) : (it.I += 1) {
...@@ -4839,42 +4849,59 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {...@@ -4839,42 +4849,59 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
4839 }4849 }
48404850
4841 const begin_c = ZigClangSourceManager_getCharacterData(c.source_manager, begin_loc);4851 const begin_c = ZigClangSourceManager_getCharacterData(c.source_manager, begin_loc);
4842 ctok.tokenizeCMacro(c, begin_loc, mangled_name, &tok_list, begin_c) catch |err| switch (err) {4852 const slice = begin_c[0..mem.len(u8, begin_c)];
4843 error.OutOfMemory => |e| return e,4853
4844 else => {4854 tok_list.shrink(0);
4845 continue;4855 var tokenizer = std.c.Tokenizer{
4856 .source = &std.c.tokenizer.Source{
4857 .buffer = slice,
4858 .file_name = undefined,
4859 .tokens = undefined,
4846 },4860 },
4847 };4861 };
4862 while (true) {
4863 const tok = tokenizer.next();
4864 switch (tok.id) {
4865 .Nl, .Eof => {
4866 try tok_list.push(tok);
4867 break;
4868 },
4869 .LineComment, .MultiLineComment => continue,
4870 else => {},
4871 }
4872 try tok_list.push(tok);
4873 }
48484874
4849 var tok_it = tok_list.iterator(0);4875 var tok_it = tok_list.iterator(0);
4850 const first_tok = tok_it.next().?;4876 const first_tok = tok_it.next().?;
4851 assert(first_tok.id == .Identifier and mem.eql(u8, first_tok.bytes, name));4877 assert(first_tok.id == .Identifier and mem.eql(u8, slice[first_tok.start..first_tok.end], name));
4878
4879 var macro_fn = false;
4852 const next = tok_it.peek().?;4880 const next = tok_it.peek().?;
4853 switch (next.id) {4881 switch (next.id) {
4854 .Identifier => {4882 .Identifier => {
4855 // if it equals itself, ignore. for example, from stdio.h:4883 // if it equals itself, ignore. for example, from stdio.h:
4856 // #define stdin stdin4884 // #define stdin stdin
4857 if (mem.eql(u8, name, next.bytes)) {4885 if (mem.eql(u8, name, slice[next.start..next.end])) {
4858 continue;4886 continue;
4859 }4887 }
4860 },4888 },
4861 .Eof => {4889 .Nl, .Eof => {
4862 // this means it is a macro without a value4890 // this means it is a macro without a value
4863 // we don't care about such things4891 // we don't care about such things
4864 continue;4892 continue;
4865 },4893 },
4894 .LParen => {
4895 // if the name is immediately followed by a '(' then it is a function
4896 macro_fn = first_tok.end == next.start;
4897 },
4866 else => {},4898 else => {},
4867 }4899 }
48684900
4869 const macro_fn = if (tok_it.peek().?.id == .Fn) blk: {
4870 _ = tok_it.next();
4871 break :blk true;
4872 } else false;
4873
4874 (if (macro_fn)4901 (if (macro_fn)
4875 transMacroFnDefine(c, &tok_it, mangled_name, begin_loc)4902 transMacroFnDefine(c, &tok_it, slice, mangled_name, begin_loc)
4876 else4903 else
4877 transMacroDefine(c, &tok_it, mangled_name, begin_loc)) catch |err| switch (err) {4904 transMacroDefine(c, &tok_it, slice, mangled_name, begin_loc)) catch |err| switch (err) {
4878 error.ParseError => continue,4905 error.ParseError => continue,
4879 error.OutOfMemory => |e| return e,4906 error.OutOfMemory => |e| return e,
4880 };4907 };
...@@ -4884,15 +4911,15 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {...@@ -4884,15 +4911,15 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
4884 }4911 }
4885}4912}
48864913
4887fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {4914fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
4888 const scope = &c.global_scope.base;4915 const scope = &c.global_scope.base;
48894916
4890 const node = try transCreateNodeVarDecl(c, true, true, name);4917 const node = try transCreateNodeVarDecl(c, true, true, name);
4891 node.eq_token = try appendToken(c, .Equal, "=");4918 node.eq_token = try appendToken(c, .Equal, "=");
48924919
4893 node.init_node = try parseCExpr(c, it, source_loc, scope);4920 node.init_node = try parseCExpr(c, it, source, source_loc, scope);
4894 const last = it.next().?;4921 const last = it.next().?;
4895 if (last.id != .Eof)4922 if (last.id != .Eof and last.id != .Nl)
4896 return failDecl(4923 return failDecl(
4897 c,4924 c,
4898 source_loc,4925 source_loc,
...@@ -4905,7 +4932,7 @@ fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8,...@@ -4905,7 +4932,7 @@ fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8,
4905 _ = try c.global_scope.macro_table.put(name, &node.base);4932 _ = try c.global_scope.macro_table.put(name, &node.base);
4906}4933}
49074934
4908fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {4935fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
4909 const block_scope = try Scope.Block.init(c, &c.global_scope.base, null);4936 const block_scope = try Scope.Block.init(c, &c.global_scope.base, null);
4910 const scope = &block_scope.base;4937 const scope = &block_scope.base;
49114938
...@@ -4937,7 +4964,7 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u...@@ -4937,7 +4964,7 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u
4937 );4964 );
4938 }4965 }
49394966
4940 const mangled_name = try block_scope.makeMangledName(c, param_tok.bytes);4967 const mangled_name = try block_scope.makeMangledName(c, source[param_tok.start..param_tok.end]);
4941 const param_name_tok = try appendIdentifier(c, mangled_name);4968 const param_name_tok = try appendIdentifier(c, mangled_name);
4942 _ = try appendToken(c, .Colon, ":");4969 _ = try appendToken(c, .Colon, ":");
49434970
...@@ -5000,9 +5027,9 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u...@@ -5000,9 +5027,9 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u
5000 const block = try transCreateNodeBlock(c, null);5027 const block = try transCreateNodeBlock(c, null);
50015028
5002 const return_expr = try transCreateNodeReturnExpr(c);5029 const return_expr = try transCreateNodeReturnExpr(c);
5003 const expr = try parseCExpr(c, it, source_loc, scope);5030 const expr = try parseCExpr(c, it, source, source_loc, scope);
5004 const last = it.next().?;5031 const last = it.next().?;
5005 if (last.id != .Eof)5032 if (last.id != .Eof and last.id != .Nl)
5006 return failDecl(5033 return failDecl(
5007 c,5034 c,
5008 source_loc,5035 source_loc,
...@@ -5022,27 +5049,28 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u...@@ -5022,27 +5049,28 @@ fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u
50225049
5023const ParseError = Error || error{ParseError};5050const ParseError = Error || error{ParseError};
50245051
5025fn parseCExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {5052fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5026 const node = try parseCPrefixOpExpr(c, it, source_loc, scope);5053 const node = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5027 switch (it.next().?.id) {5054 switch (it.next().?.id) {
5028 .QuestionMark => {5055 .QuestionMark => {
5029 // must come immediately after expr5056 // must come immediately after expr
5030 _ = try appendToken(c, .RParen, ")");5057 _ = try appendToken(c, .RParen, ")");
5031 const if_node = try transCreateNodeIf(c);5058 const if_node = try transCreateNodeIf(c);
5032 if_node.condition = node;5059 if_node.condition = node;
5033 if_node.body = try parseCPrimaryExpr(c, it, source_loc, scope);5060 if_node.body = try parseCPrimaryExpr(c, it, source, source_loc, scope);
5034 if (it.next().?.id != .Colon) {5061 if (it.next().?.id != .Colon) {
5062 const first_tok = it.list.at(0);
5035 try failDecl(5063 try failDecl(
5036 c,5064 c,
5037 source_loc,5065 source_loc,
5038 it.list.at(0).*.bytes,5066 source[first_tok.start..first_tok.end],
5039 "unable to translate C expr: expected ':'",5067 "unable to translate C expr: expected ':'",
5040 .{},5068 .{},
5041 );5069 );
5042 return error.ParseError;5070 return error.ParseError;
5043 }5071 }
5044 if_node.@"else" = try transCreateNodeElse(c);5072 if_node.@"else" = try transCreateNodeElse(c);
5045 if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source_loc, scope);5073 if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source, source_loc, scope);
5046 return &if_node.base;5074 return &if_node.base;
5047 },5075 },
5048 else => {5076 else => {
...@@ -5052,30 +5080,30 @@ fn parseCExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSou...@@ -5052,30 +5080,30 @@ fn parseCExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSou
5052 }5080 }
5053}5081}
50545082
5055fn parseCNumLit(c: *Context, tok: *CToken, source_loc: ZigClangSourceLocation) ParseError!*ast.Node {5083fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigClangSourceLocation) ParseError!*ast.Node {
5056 if (tok.id == .NumLitInt) {5084 var lit_bytes = source[tok.start..tok.end];
5057 var lit_bytes = tok.bytes;
50585085
5059 if (tok.bytes.len > 2 and tok.bytes[0] == '0') {5086 if (tok.id == .IntegerLiteral) {
5060 switch (tok.bytes[1]) {5087 if (lit_bytes.len > 2 and lit_bytes[0] == '0') {
5088 switch (lit_bytes[1]) {
5061 '0'...'7' => {5089 '0'...'7' => {
5062 // Octal5090 // Octal
5063 lit_bytes = try std.fmt.allocPrint(c.a(), "0o{}", .{tok.bytes});5091 lit_bytes = try std.fmt.allocPrint(c.a(), "0o{}", .{lit_bytes});
5064 },5092 },
5065 'X' => {5093 'X' => {
5066 // Hexadecimal with capital X, valid in C but not in Zig5094 // Hexadecimal with capital X, valid in C but not in Zig
5067 lit_bytes = try std.fmt.allocPrint(c.a(), "0x{}", .{tok.bytes[2..]});5095 lit_bytes = try std.fmt.allocPrint(c.a(), "0x{}", .{lit_bytes[2..]});
5068 },5096 },
5069 else => {},5097 else => {},
5070 }5098 }
5071 }5099 }
50725100
5073 if (tok.num_lit_suffix == .None) {5101 if (tok.id.IntegerLiteral == .None) {
5074 return transCreateNodeInt(c, lit_bytes);5102 return transCreateNodeInt(c, lit_bytes);
5075 }5103 }
50765104
5077 const cast_node = try transCreateNodeBuiltinFnCall(c, "@as");5105 const cast_node = try transCreateNodeBuiltinFnCall(c, "@as");
5078 try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.num_lit_suffix) {5106 try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.id.IntegerLiteral) {
5079 .U => "c_uint",5107 .U => "c_uint",
5080 .L => "c_long",5108 .L => "c_long",
5081 .LU => "c_ulong",5109 .LU => "c_ulong",
...@@ -5083,55 +5111,233 @@ fn parseCNumLit(c: *Context, tok: *CToken, source_loc: ZigClangSourceLocation) P...@@ -5083,55 +5111,233 @@ fn parseCNumLit(c: *Context, tok: *CToken, source_loc: ZigClangSourceLocation) P
5083 .LLU => "c_ulonglong",5111 .LLU => "c_ulonglong",
5084 else => unreachable,5112 else => unreachable,
5085 }));5113 }));
5114 lit_bytes = lit_bytes[0 .. lit_bytes.len - switch (tok.id.IntegerLiteral) {
5115 .U, .L => @as(u8, 1),
5116 .LU, .LL => 2,
5117 .LLU => 3,
5118 else => unreachable,
5119 }];
5086 _ = try appendToken(c, .Comma, ",");5120 _ = try appendToken(c, .Comma, ",");
5087 try cast_node.params.push(try transCreateNodeInt(c, lit_bytes));5121 try cast_node.params.push(try transCreateNodeInt(c, lit_bytes));
5088 cast_node.rparen_token = try appendToken(c, .RParen, ")");5122 cast_node.rparen_token = try appendToken(c, .RParen, ")");
5089 return &cast_node.base;5123 return &cast_node.base;
5090 } else if (tok.id == .NumLitFloat) {5124 } else if (tok.id == .FloatLiteral) {
5091 if (tok.num_lit_suffix == .None) {5125 if (tok.id.FloatLiteral == .None) {
5092 return transCreateNodeFloat(c, tok.bytes);5126 return transCreateNodeFloat(c, lit_bytes);
5093 }5127 }
5094 const cast_node = try transCreateNodeBuiltinFnCall(c, "@as");5128 const cast_node = try transCreateNodeBuiltinFnCall(c, "@as");
5095 try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.num_lit_suffix) {5129 try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.id.FloatLiteral) {
5096 .F => "f32",5130 .F => "f32",
5097 .L => "f64",5131 .L => "c_longdouble",
5098 else => unreachable,5132 else => unreachable,
5099 }));5133 }));
5100 _ = try appendToken(c, .Comma, ",");5134 _ = try appendToken(c, .Comma, ",");
5101 try cast_node.params.push(try transCreateNodeFloat(c, tok.bytes));5135 try cast_node.params.push(try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1]));
5102 cast_node.rparen_token = try appendToken(c, .RParen, ")");5136 cast_node.rparen_token = try appendToken(c, .RParen, ")");
5103 return &cast_node.base;5137 return &cast_node.base;
5104 } else unreachable;5138 } else unreachable;
5105}5139}
51065140
5107fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {5141fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ![]const u8 {
5142 var source = source_bytes;
5143 for (source) |c, i| {
5144 if (c == '\"' or c == '\'') {
5145 source = source[i..];
5146 break;
5147 }
5148 }
5149 for (source) |c| {
5150 if (c == '\\') {
5151 break;
5152 }
5153 } else return source;
5154 var bytes = try ctx.a().alloc(u8, source.len * 2);
5155 var state: enum {
5156 Start,
5157 Escape,
5158 Hex,
5159 Octal,
5160 } = .Start;
5161 var i: usize = 0;
5162 var count: u8 = 0;
5163 var num: u8 = 0;
5164 for (source) |c| {
5165 switch (state) {
5166 .Escape => {
5167 switch (c) {
5168 'n', 'r', 't', '\\', '\'', '\"' => {
5169 bytes[i] = c;
5170 },
5171 '0'...'7' => {
5172 count += 1;
5173 num += c - '0';
5174 state = .Octal;
5175 bytes[i] = 'x';
5176 },
5177 'x' => {
5178 state = .Hex;
5179 bytes[i] = 'x';
5180 },
5181 'a' => {
5182 bytes[i] = 'x';
5183 i += 1;
5184 bytes[i] = '0';
5185 i += 1;
5186 bytes[i] = '7';
5187 },
5188 'b' => {
5189 bytes[i] = 'x';
5190 i += 1;
5191 bytes[i] = '0';
5192 i += 1;
5193 bytes[i] = '8';
5194 },
5195 'f' => {
5196 bytes[i] = 'x';
5197 i += 1;
5198 bytes[i] = '0';
5199 i += 1;
5200 bytes[i] = 'C';
5201 },
5202 'v' => {
5203 bytes[i] = 'x';
5204 i += 1;
5205 bytes[i] = '0';
5206 i += 1;
5207 bytes[i] = 'B';
5208 },
5209 '?' => {
5210 i -= 1;
5211 bytes[i] = '?';
5212 },
5213 'u', 'U' => {
5214 try failDecl(ctx, source_loc, name, "macro tokenizing failed: TODO unicode escape sequences", .{});
5215 return error.ParseError;
5216 },
5217 else => {
5218 try failDecl(ctx, source_loc, name, "macro tokenizing failed: unknown escape sequence", .{});
5219 return error.ParseError;
5220 },
5221 }
5222 i += 1;
5223 if (state == .Escape)
5224 state = .Start;
5225 },
5226 .Start => {
5227 if (c == '\\') {
5228 state = .Escape;
5229 }
5230 bytes[i] = c;
5231 i += 1;
5232 },
5233 .Hex => {
5234 switch (c) {
5235 '0'...'9' => {
5236 num = std.math.mul(u8, num, 16) catch {
5237 try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
5238 return error.ParseError;
5239 };
5240 num += c - '0';
5241 },
5242 'a'...'f' => {
5243 num = std.math.mul(u8, num, 16) catch {
5244 try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
5245 return error.ParseError;
5246 };
5247 num += c - 'a' + 10;
5248 },
5249 'A'...'F' => {
5250 num = std.math.mul(u8, num, 16) catch {
5251 try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
5252 return error.ParseError;
5253 };
5254 num += c - 'A' + 10;
5255 },
5256 else => {
5257 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
5258 num = 0;
5259 if (c == '\\')
5260 state = .Escape
5261 else
5262 state = .Start;
5263 bytes[i] = c;
5264 i += 1;
5265 },
5266 }
5267 },
5268 .Octal => {
5269 const accept_digit = switch (c) {
5270 // The maximum length of a octal literal is 3 digits
5271 '0'...'7' => count < 3,
5272 else => false,
5273 };
5274
5275 if (accept_digit) {
5276 count += 1;
5277 num = std.math.mul(u8, num, 8) catch {
5278 try failDecl(ctx, source_loc, name, "macro tokenizing failed: octal literal overflowed", .{});
5279 return error.ParseError;
5280 };
5281 num += c - '0';
5282 } else {
5283 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
5284 num = 0;
5285 count = 0;
5286 if (c == '\\')
5287 state = .Escape
5288 else
5289 state = .Start;
5290 bytes[i] = c;
5291 i += 1;
5292 }
5293 },
5294 }
5295 }
5296 if (state == .Hex or state == .Octal)
5297 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
5298 return bytes[0..i];
5299}
5300
5301fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5108 const tok = it.next().?;5302 const tok = it.next().?;
5109 switch (tok.id) {5303 switch (tok.id) {
5110 .CharLit => {5304 .CharLiteral => {
5111 const token = try appendToken(c, .CharLiteral, tok.bytes);5305 const first_tok = it.list.at(0);
5306 const token = try appendToken(c, .CharLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc));
5112 const node = try c.a().create(ast.Node.CharLiteral);5307 const node = try c.a().create(ast.Node.CharLiteral);
5113 node.* = ast.Node.CharLiteral{5308 node.* = ast.Node.CharLiteral{
5114 .token = token,5309 .token = token,
5115 };5310 };
5116 return &node.base;5311 return &node.base;
5117 },5312 },
5118 .StrLit => {5313 .StringLiteral => {
5119 const token = try appendToken(c, .StringLiteral, tok.bytes);5314 const first_tok = it.list.at(0);
5315 const token = try appendToken(c, .StringLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc));
5120 const node = try c.a().create(ast.Node.StringLiteral);5316 const node = try c.a().create(ast.Node.StringLiteral);
5121 node.* = ast.Node.StringLiteral{5317 node.* = ast.Node.StringLiteral{
5122 .token = token,5318 .token = token,
5123 };5319 };
5124 return &node.base;5320 return &node.base;
5125 },5321 },
5126 .NumLitInt, .NumLitFloat => {5322 .IntegerLiteral, .FloatLiteral => {
5127 return parseCNumLit(c, tok, source_loc);5323 return parseCNumLit(c, tok, source, source_loc);
5128 },5324 },
5325 // eventually this will be replaced by std.c.parse which will handle these correctly
5326 .Keyword_void => return transCreateNodeTypeIdentifier(c, "c_void"),
5327 .Keyword_bool => return transCreateNodeTypeIdentifier(c, "bool"),
5328 .Keyword_double => return transCreateNodeTypeIdentifier(c, "f64"),
5329 .Keyword_long => return transCreateNodeTypeIdentifier(c, "c_long"),
5330 .Keyword_int => return transCreateNodeTypeIdentifier(c, "c_int"),
5331 .Keyword_float => return transCreateNodeTypeIdentifier(c, "f32"),
5332 .Keyword_short => return transCreateNodeTypeIdentifier(c, "c_short"),
5333 .Keyword_char => return transCreateNodeTypeIdentifier(c, "c_char"),
5334 .Keyword_unsigned => return transCreateNodeTypeIdentifier(c, "c_uint"),
5129 .Identifier => {5335 .Identifier => {
5130 const mangled_name = scope.getAlias(tok.bytes);5336 const mangled_name = scope.getAlias(source[tok.start..tok.end]);
5131 return transCreateNodeIdentifier(c, mangled_name);5337 return transCreateNodeIdentifier(c, mangled_name);
5132 },5338 },
5133 .LParen => {5339 .LParen => {
5134 const inner_node = try parseCExpr(c, it, source_loc, scope);5340 const inner_node = try parseCExpr(c, it, source, source_loc, scope);
51355341
5136 if (it.peek().?.id == .RParen) {5342 if (it.peek().?.id == .RParen) {
5137 _ = it.next();5343 _ = it.next();
...@@ -5144,13 +5350,14 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC...@@ -5144,13 +5350,14 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC
5144 // hack to get zig fmt to render a comma in builtin calls5350 // hack to get zig fmt to render a comma in builtin calls
5145 _ = try appendToken(c, .Comma, ",");5351 _ = try appendToken(c, .Comma, ",");
51465352
5147 const node_to_cast = try parseCExpr(c, it, source_loc, scope);5353 const node_to_cast = try parseCExpr(c, it, source, source_loc, scope);
51485354
5149 if (it.next().?.id != .RParen) {5355 if (it.next().?.id != .RParen) {
5356 const first_tok = it.list.at(0);
5150 try failDecl(5357 try failDecl(
5151 c,5358 c,
5152 source_loc,5359 source_loc,
5153 it.list.at(0).*.bytes,5360 source[first_tok.start..first_tok.end],
5154 "unable to translate C expr: expected ')''",5361 "unable to translate C expr: expected ')''",
5155 .{},5362 .{},
5156 );5363 );
...@@ -5228,10 +5435,11 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC...@@ -5228,10 +5435,11 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC
5228 return &if_1.base;5435 return &if_1.base;
5229 },5436 },
5230 else => {5437 else => {
5438 const first_tok = it.list.at(0);
5231 try failDecl(5439 try failDecl(
5232 c,5440 c,
5233 source_loc,5441 source_loc,
5234 it.list.at(0).*.bytes,5442 source[first_tok.start..first_tok.end],
5235 "unable to translate C expr: unexpected token {}",5443 "unable to translate C expr: unexpected token {}",
5236 .{tok.id},5444 .{tok.id},
5237 );5445 );
...@@ -5240,33 +5448,35 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC...@@ -5240,33 +5448,35 @@ fn parseCPrimaryExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigC
5240 }5448 }
5241}5449}
52425450
5243fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {5451fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5244 var node = try parseCPrimaryExpr(c, it, source_loc, scope);5452 var node = try parseCPrimaryExpr(c, it, source, source_loc, scope);
5245 while (true) {5453 while (true) {
5246 const tok = it.next().?;5454 const tok = it.next().?;
5247 switch (tok.id) {5455 switch (tok.id) {
5248 .Dot => {5456 .Period => {
5249 const name_tok = it.next().?;5457 const name_tok = it.next().?;
5250 if (name_tok.id != .Identifier) {5458 if (name_tok.id != .Identifier) {
5459 const first_tok = it.list.at(0);
5251 try failDecl(5460 try failDecl(
5252 c,5461 c,
5253 source_loc,5462 source_loc,
5254 it.list.at(0).*.bytes,5463 source[first_tok.start..first_tok.end],
5255 "unable to translate C expr: expected identifier",5464 "unable to translate C expr: expected identifier",
5256 .{},5465 .{},
5257 );5466 );
5258 return error.ParseError;5467 return error.ParseError;
5259 }5468 }
52605469
5261 node = try transCreateNodeFieldAccess(c, node, name_tok.bytes);5470 node = try transCreateNodeFieldAccess(c, node, source[name_tok.start..name_tok.end]);
5262 },5471 },
5263 .Arrow => {5472 .Arrow => {
5264 const name_tok = it.next().?;5473 const name_tok = it.next().?;
5265 if (name_tok.id != .Identifier) {5474 if (name_tok.id != .Identifier) {
5475 const first_tok = it.list.at(0);
5266 try failDecl(5476 try failDecl(
5267 c,5477 c,
5268 source_loc,5478 source_loc,
5269 it.list.at(0).*.bytes,5479 source[first_tok.start..first_tok.end],
5270 "unable to translate C expr: expected identifier",5480 "unable to translate C expr: expected identifier",
5271 .{},5481 .{},
5272 );5482 );
...@@ -5274,7 +5484,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5274,7 +5484,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5274 }5484 }
52755485
5276 const deref = try transCreateNodePtrDeref(c, node);5486 const deref = try transCreateNodePtrDeref(c, node);
5277 node = try transCreateNodeFieldAccess(c, deref, name_tok.bytes);5487 node = try transCreateNodeFieldAccess(c, deref, source[name_tok.start..name_tok.end]);
5278 },5488 },
5279 .Asterisk => {5489 .Asterisk => {
5280 if (it.peek().?.id == .RParen) {5490 if (it.peek().?.id == .RParen) {
...@@ -5283,13 +5493,23 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5283,13 +5493,23 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5283 // hack to get zig fmt to render a comma in builtin calls5493 // hack to get zig fmt to render a comma in builtin calls
5284 _ = try appendToken(c, .Comma, ",");5494 _ = try appendToken(c, .Comma, ",");
52855495
5286 const ptr = try transCreateNodePtrType(c, false, false, .Identifier);5496 const ptr_kind = blk:{
5497 // * token
5498 _ = it.prev();
5499 // last token of `node`
5500 const prev_id = it.prev().?.id;
5501 _ = it.next();
5502 _ = it.next();
5503 break :blk if (prev_id == .Keyword_void) .Asterisk else Token.Id.Identifier;
5504 };
5505
5506 const ptr = try transCreateNodePtrType(c, false, false, ptr_kind);
5287 ptr.rhs = node;5507 ptr.rhs = node;
5288 return &ptr.base;5508 return &ptr.base;
5289 } else {5509 } else {
5290 // expr * expr5510 // expr * expr
5291 const op_token = try appendToken(c, .Asterisk, "*");5511 const op_token = try appendToken(c, .Asterisk, "*");
5292 const rhs = try parseCPrimaryExpr(c, it, source_loc, scope);5512 const rhs = try parseCPrimaryExpr(c, it, source, source_loc, scope);
5293 const mul_node = try c.a().create(ast.Node.InfixOp);5513 const mul_node = try c.a().create(ast.Node.InfixOp);
5294 mul_node.* = .{5514 mul_node.* = .{
5295 .op_token = op_token,5515 .op_token = op_token,
...@@ -5300,9 +5520,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5300,9 +5520,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5300 node = &mul_node.base;5520 node = &mul_node.base;
5301 }5521 }
5302 },5522 },
5303 .Shl => {5523 .AngleBracketAngleBracketLeft => {
5304 const op_token = try appendToken(c, .AngleBracketAngleBracketLeft, "<<");5524 const op_token = try appendToken(c, .AngleBracketAngleBracketLeft, "<<");
5305 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5525 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5306 const bitshift_node = try c.a().create(ast.Node.InfixOp);5526 const bitshift_node = try c.a().create(ast.Node.InfixOp);
5307 bitshift_node.* = .{5527 bitshift_node.* = .{
5308 .op_token = op_token,5528 .op_token = op_token,
...@@ -5312,9 +5532,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5312,9 +5532,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5312 };5532 };
5313 node = &bitshift_node.base;5533 node = &bitshift_node.base;
5314 },5534 },
5315 .Shr => {5535 .AngleBracketAngleBracketRight => {
5316 const op_token = try appendToken(c, .AngleBracketAngleBracketRight, ">>");5536 const op_token = try appendToken(c, .AngleBracketAngleBracketRight, ">>");
5317 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5537 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5318 const bitshift_node = try c.a().create(ast.Node.InfixOp);5538 const bitshift_node = try c.a().create(ast.Node.InfixOp);
5319 bitshift_node.* = .{5539 bitshift_node.* = .{
5320 .op_token = op_token,5540 .op_token = op_token,
...@@ -5326,7 +5546,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5326,7 +5546,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5326 },5546 },
5327 .Pipe => {5547 .Pipe => {
5328 const op_token = try appendToken(c, .Pipe, "|");5548 const op_token = try appendToken(c, .Pipe, "|");
5329 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5549 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5330 const or_node = try c.a().create(ast.Node.InfixOp);5550 const or_node = try c.a().create(ast.Node.InfixOp);
5331 or_node.* = .{5551 or_node.* = .{
5332 .op_token = op_token,5552 .op_token = op_token,
...@@ -5338,7 +5558,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5338,7 +5558,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5338 },5558 },
5339 .Ampersand => {5559 .Ampersand => {
5340 const op_token = try appendToken(c, .Ampersand, "&");5560 const op_token = try appendToken(c, .Ampersand, "&");
5341 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5561 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5342 const bitand_node = try c.a().create(ast.Node.InfixOp);5562 const bitand_node = try c.a().create(ast.Node.InfixOp);
5343 bitand_node.* = .{5563 bitand_node.* = .{
5344 .op_token = op_token,5564 .op_token = op_token,
...@@ -5350,7 +5570,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5350,7 +5570,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5350 },5570 },
5351 .Plus => {5571 .Plus => {
5352 const op_token = try appendToken(c, .Plus, "+");5572 const op_token = try appendToken(c, .Plus, "+");
5353 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5573 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5354 const add_node = try c.a().create(ast.Node.InfixOp);5574 const add_node = try c.a().create(ast.Node.InfixOp);
5355 add_node.* = .{5575 add_node.* = .{
5356 .op_token = op_token,5576 .op_token = op_token,
...@@ -5362,7 +5582,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5362,7 +5582,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5362 },5582 },
5363 .Minus => {5583 .Minus => {
5364 const op_token = try appendToken(c, .Minus, "-");5584 const op_token = try appendToken(c, .Minus, "-");
5365 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5585 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5366 const sub_node = try c.a().create(ast.Node.InfixOp);5586 const sub_node = try c.a().create(ast.Node.InfixOp);
5367 sub_node.* = .{5587 sub_node.* = .{
5368 .op_token = op_token,5588 .op_token = op_token,
...@@ -5372,9 +5592,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5372,9 +5592,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5372 };5592 };
5373 node = &sub_node.base;5593 node = &sub_node.base;
5374 },5594 },
5375 .And => {5595 .AmpersandAmpersand => {
5376 const op_token = try appendToken(c, .Keyword_and, "and");5596 const op_token = try appendToken(c, .Keyword_and, "and");
5377 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5597 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5378 const and_node = try c.a().create(ast.Node.InfixOp);5598 const and_node = try c.a().create(ast.Node.InfixOp);
5379 and_node.* = .{5599 and_node.* = .{
5380 .op_token = op_token,5600 .op_token = op_token,
...@@ -5384,9 +5604,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5384,9 +5604,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5384 };5604 };
5385 node = &and_node.base;5605 node = &and_node.base;
5386 },5606 },
5387 .Or => {5607 .PipePipe => {
5388 const op_token = try appendToken(c, .Keyword_or, "or");5608 const op_token = try appendToken(c, .Keyword_or, "or");
5389 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5609 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5390 const or_node = try c.a().create(ast.Node.InfixOp);5610 const or_node = try c.a().create(ast.Node.InfixOp);
5391 or_node.* = .{5611 or_node.* = .{
5392 .op_token = op_token,5612 .op_token = op_token,
...@@ -5396,9 +5616,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5396,9 +5616,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5396 };5616 };
5397 node = &or_node.base;5617 node = &or_node.base;
5398 },5618 },
5399 .Gt => {5619 .AngleBracketRight => {
5400 const op_token = try appendToken(c, .AngleBracketRight, ">");5620 const op_token = try appendToken(c, .AngleBracketRight, ">");
5401 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5621 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5402 const and_node = try c.a().create(ast.Node.InfixOp);5622 const and_node = try c.a().create(ast.Node.InfixOp);
5403 and_node.* = .{5623 and_node.* = .{
5404 .op_token = op_token,5624 .op_token = op_token,
...@@ -5408,9 +5628,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5408,9 +5628,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5408 };5628 };
5409 node = &and_node.base;5629 node = &and_node.base;
5410 },5630 },
5411 .Gte => {5631 .AngleBracketRightEqual => {
5412 const op_token = try appendToken(c, .AngleBracketRightEqual, ">=");5632 const op_token = try appendToken(c, .AngleBracketRightEqual, ">=");
5413 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5633 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5414 const and_node = try c.a().create(ast.Node.InfixOp);5634 const and_node = try c.a().create(ast.Node.InfixOp);
5415 and_node.* = .{5635 and_node.* = .{
5416 .op_token = op_token,5636 .op_token = op_token,
...@@ -5420,9 +5640,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5420,9 +5640,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5420 };5640 };
5421 node = &and_node.base;5641 node = &and_node.base;
5422 },5642 },
5423 .Lt => {5643 .AngleBracketLeft => {
5424 const op_token = try appendToken(c, .AngleBracketLeft, "<");5644 const op_token = try appendToken(c, .AngleBracketLeft, "<");
5425 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5645 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5426 const and_node = try c.a().create(ast.Node.InfixOp);5646 const and_node = try c.a().create(ast.Node.InfixOp);
5427 and_node.* = .{5647 and_node.* = .{
5428 .op_token = op_token,5648 .op_token = op_token,
...@@ -5432,9 +5652,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5432,9 +5652,9 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5432 };5652 };
5433 node = &and_node.base;5653 node = &and_node.base;
5434 },5654 },
5435 .Lte => {5655 .AngleBracketLeftEqual => {
5436 const op_token = try appendToken(c, .AngleBracketLeftEqual, "<=");5656 const op_token = try appendToken(c, .AngleBracketLeftEqual, "<=");
5437 const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5657 const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5438 const and_node = try c.a().create(ast.Node.InfixOp);5658 const and_node = try c.a().create(ast.Node.InfixOp);
5439 and_node.* = .{5659 and_node.* = .{
5440 .op_token = op_token,5660 .op_token = op_token,
...@@ -5444,16 +5664,17 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5444,16 +5664,17 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5444 };5664 };
5445 node = &and_node.base;5665 node = &and_node.base;
5446 },5666 },
5447 .LBrace => {5667 .LBracket => {
5448 const arr_node = try transCreateNodeArrayAccess(c, node);5668 const arr_node = try transCreateNodeArrayAccess(c, node);
5449 arr_node.op.ArrayAccess = try parseCPrefixOpExpr(c, it, source_loc, scope);5669 arr_node.op.ArrayAccess = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5450 arr_node.rtoken = try appendToken(c, .RBrace, "]");5670 arr_node.rtoken = try appendToken(c, .RBracket, "]");
5451 node = &arr_node.base;5671 node = &arr_node.base;
5452 if (it.next().?.id != .RBrace) {5672 if (it.next().?.id != .RBracket) {
5673 const first_tok = it.list.at(0);
5453 try failDecl(5674 try failDecl(
5454 c,5675 c,
5455 source_loc,5676 source_loc,
5456 it.list.at(0).*.bytes,5677 source[first_tok.start..first_tok.end],
5457 "unable to translate C expr: expected ']'",5678 "unable to translate C expr: expected ']'",
5458 .{},5679 .{},
5459 );5680 );
...@@ -5463,7 +5684,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5463,7 +5684,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5463 .LParen => {5684 .LParen => {
5464 const call_node = try transCreateNodeFnCall(c, node);5685 const call_node = try transCreateNodeFnCall(c, node);
5465 while (true) {5686 while (true) {
5466 const arg = try parseCPrefixOpExpr(c, it, source_loc, scope);5687 const arg = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5467 try call_node.op.Call.params.push(arg);5688 try call_node.op.Call.params.push(arg);
5468 const next = it.next().?;5689 const next = it.next().?;
5469 if (next.id == .Comma)5690 if (next.id == .Comma)
...@@ -5471,10 +5692,11 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5471,10 +5692,11 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5471 else if (next.id == .RParen)5692 else if (next.id == .RParen)
5472 break5693 break
5473 else {5694 else {
5695 const first_tok = it.list.at(0);
5474 try failDecl(5696 try failDecl(
5475 c,5697 c,
5476 source_loc,5698 source_loc,
5477 it.list.at(0).*.bytes,5699 source[first_tok.start..first_tok.end],
5478 "unable to translate C expr: expected ',' or ')'",5700 "unable to translate C expr: expected ',' or ')'",
5479 .{},5701 .{},
5480 );5702 );
...@@ -5492,32 +5714,32 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig...@@ -5492,32 +5714,32 @@ fn parseCSuffixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: Zig
5492 }5714 }
5493}5715}
54945716
5495fn parseCPrefixOpExpr(c: *Context, it: *ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {5717fn parseCPrefixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: *Scope) ParseError!*ast.Node {
5496 const op_tok = it.next().?;5718 const op_tok = it.next().?;
54975719
5498 switch (op_tok.id) {5720 switch (op_tok.id) {
5499 .Bang => {5721 .Bang => {
5500 const node = try transCreateNodePrefixOp(c, .BoolNot, .Bang, "!");5722 const node = try transCreateNodePrefixOp(c, .BoolNot, .Bang, "!");
5501 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5723 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5502 return &node.base;5724 return &node.base;
5503 },5725 },
5504 .Minus => {5726 .Minus => {
5505 const node = try transCreateNodePrefixOp(c, .Negation, .Minus, "-");5727 const node = try transCreateNodePrefixOp(c, .Negation, .Minus, "-");
5506 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5728 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5507 return &node.base;5729 return &node.base;
5508 },5730 },
5509 .Tilde => {5731 .Tilde => {
5510 const node = try transCreateNodePrefixOp(c, .BitNot, .Tilde, "~");5732 const node = try transCreateNodePrefixOp(c, .BitNot, .Tilde, "~");
5511 node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);5733 node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5512 return &node.base;5734 return &node.base;
5513 },5735 },
5514 .Asterisk => {5736 .Asterisk => {
5515 const prefix_op_expr = try parseCPrefixOpExpr(c, it, source_loc, scope);5737 const prefix_op_expr = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5516 return try transCreateNodePtrDeref(c, prefix_op_expr);5738 return try transCreateNodePtrDeref(c, prefix_op_expr);
5517 },5739 },
5518 else => {5740 else => {
5519 _ = it.prev();5741 _ = it.prev();
5520 return try parseCSuffixOpExpr(c, it, source_loc, scope);5742 return try parseCSuffixOpExpr(c, it, source, source_loc, scope);
5521 },5743 },
5522 }5744 }
5523}5745}
test/translate_c.zig+24-2
...@@ -618,6 +618,28 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -618,6 +618,28 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
618 },618 },
619 );619 );
620620
621 cases.add("float suffixes",
622 \\#define foo 3.14f
623 \\#define bar 16.e-2l
624 , &[_][]const u8{
625 "pub const foo = @as(f32, 3.14);",
626 "pub const bar = @as(c_longdouble, 16.e-2);",
627 });
628
629 cases.add("comments",
630 \\#define foo 1 //foo
631 \\#define bar /* bar */ 2
632 , &[_][]const u8{
633 "pub const foo = 1;",
634 "pub const bar = 2;",
635 });
636
637 cases.add("string prefix",
638 \\#define foo L"hello"
639 , &[_][]const u8{
640 "pub const foo = \"hello\";",
641 });
642
621 cases.add("null statements",643 cases.add("null statements",
622 \\void foo(void) {644 \\void foo(void) {
623 \\ ;;;;;645 \\ ;;;;;
...@@ -2508,8 +2530,8 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -2508,8 +2530,8 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
2508 cases.add("macro cast",2530 cases.add("macro cast",
2509 \\#define FOO(bar) baz((void *)(baz))2531 \\#define FOO(bar) baz((void *)(baz))
2510 , &[_][]const u8{2532 , &[_][]const u8{
2511 \\pub inline fn FOO(bar: var) @TypeOf(baz(if (@typeId(@TypeOf(baz)) == .Pointer) @ptrCast([*c]void, baz) else if (@typeId(@TypeOf(baz)) == .Int) @intToPtr([*c]void, baz) else @as([*c]void, baz))) {2533 \\pub inline fn FOO(bar: var) @TypeOf(baz(if (@typeId(@TypeOf(baz)) == .Pointer) @ptrCast(*c_void, baz) else if (@typeId(@TypeOf(baz)) == .Int) @intToPtr(*c_void, baz) else @as(*c_void, baz))) {
2512 \\ return baz(if (@typeId(@TypeOf(baz)) == .Pointer) @ptrCast([*c]void, baz) else if (@typeId(@TypeOf(baz)) == .Int) @intToPtr([*c]void, baz) else @as([*c]void, baz));2534 \\ return baz(if (@typeId(@TypeOf(baz)) == .Pointer) @ptrCast(*c_void, baz) else if (@typeId(@TypeOf(baz)) == .Int) @intToPtr(*c_void, baz) else @as(*c_void, baz));
2513 \\}2535 \\}
2514 });2536 });
25152537