Commit 40b1fecace87abc2e508bfafea955b3d08266301 | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-02-05 14:29:39-05:00
committer	GitHub <noreply@github.com> 2020-02-05 14:29:39-05:00
log	40b1fecace87abc2e508bfafea955b3d08266301
tree	5f13e57e47472a33692cf302303636ff63a3f8cb
parent	b022db16ece48f2cdec47c5716601fec2fc07ef4
parent	f196ddd251d80fac74685d822ec26a646c7890fa
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4363 from Vexu/trans-std-c

Use std.c.tokenizer in translate-c

4 files changed, 348 insertions(+), 1078 deletions(-)

lib
- std
  - c
    tokenizer.zig
src-self-hosted
- c_tokenizer.zig
- translate_c.zig
test
- translate_c.zig

lib/std/c/tokenizer.zig+4-1

...	...	@@ -776,12 +776,14 @@ pub const Tokenizer = struct {
776	776	}
777	777	},
778	778	else => {
	779	self.index -= 1;
779	780	state = if (string) .StringLiteral else .CharLiteral;
780	781	},
781	782	},
782	783	.HexEscape => switch (c) {
783	784	'0'...'9', 'a'...'f', 'A'...'F' => {},
784	785	else => {
	786	self.index -= 1;
785	787	state = if (string) .StringLiteral else .CharLiteral;
786	788	},
787	789	},
...	...	@@ -797,6 +799,7 @@ pub const Tokenizer = struct {
797	799	result.id = .Invalid;
798	800	break;
799	801	}
	802	self.index -= 1;
800	803	state = if (string) .StringLiteral else .CharLiteral;
801	804	},
802	805	},
...	...	@@ -1046,7 +1049,6 @@ pub const Tokenizer = struct {
1046	1049	.LineComment => switch (c) {
1047	1050	'\n' => {
1048	1051	result.id = .LineComment;
1049		self.index += 1;
1050	1052	break;
1051	1053	},
1052	1054	else => {},
...	...	@@ -1217,6 +1219,7 @@ pub const Tokenizer = struct {
1217	1219	result.id = .Invalid;
1218	1220	break;
1219	1221	}
	1222	self.index -= 1;
1220	1223	state = .FloatSuffix;
1221	1224	},
1222	1225	},

src-self-hosted/c_tokenizer.zig deleted-977

...	...	@@ -1,977 +0,0 @@
1		const std = @import("std");
2		const expect = std.testing.expect;
3		const ZigClangSourceLocation = @import("clang.zig").ZigClangSourceLocation;
4		const Context = @import("translate_c.zig").Context;
5		const failDecl = @import("translate_c.zig").failDecl;
6
7		pub const TokenList = std.SegmentedList(CToken, 32);
8
9		pub const CToken = struct {
10		id: Id,
11		bytes: []const u8 = "",
12		num_lit_suffix: NumLitSuffix = .None,
13
14		pub const Id = enum {
15		CharLit,
16		StrLit,
17		NumLitInt,
18		NumLitFloat,
19		Identifier,
20		Plus,
21		Minus,
22		Slash,
23		LParen,
24		RParen,
25		Eof,
26		Dot,
27		Asterisk, // *
28		Ampersand, // &
29		And, // &&
30		Assign, // =
31		Or, // \|\|
32		Bang, // !
33		Tilde, // ~
34		Shl, // <<
35		Shr, // >>
36		Lt, // <
37		Lte, // <=
38		Gt, // >
39		Gte, // >=
40		Eq, // ==
41		Ne, // !=
42		Increment, // ++
43		Decrement, // --
44		Comma,
45		Fn,
46		Arrow, // ->
47		LBrace,
48		RBrace,
49		Pipe,
50		QuestionMark,
51		Colon,
52		};
53
54		pub const NumLitSuffix = enum {
55		None,
56		F,
57		L,
58		U,
59		LU,
60		LL,
61		LLU,
62		};
63		};
64
65		pub fn tokenizeCMacro(ctx: Context, loc: ZigClangSourceLocation, name: []const u8, tl: TokenList, chars: [*:0]const u8) !void {
66		var index: usize = 0;
67		var first = true;
68		while (true) {
69		const tok = try next(ctx, loc, name, chars, &index);
70		if (tok.id == .StrLit or tok.id == .CharLit)
71		try tl.push(try zigifyEscapeSequences(ctx, loc, name, tl.allocator, tok))
72		else
73		try tl.push(tok);
74		if (tok.id == .Eof)
75		return;
76		if (first) {
77		// distinguish NAME (EXPR) from NAME(ARGS)
78		first = false;
79		if (chars[index] == '(') {
80		try tl.push(.{
81		.id = .Fn,
82		.bytes = "",
83		});
84		}
85		}
86		}
87		}
88
89		fn zigifyEscapeSequences(ctx: Context, loc: ZigClangSourceLocation, name: []const u8, allocator: std.mem.Allocator, tok: CToken) !CToken {
90		for (tok.bytes) \|c\| {
91		if (c == '\\') {
92		break;
93		}
94		} else return tok;
95		var bytes = try allocator.alloc(u8, tok.bytes.len * 2);
96		var state: enum {
97		Start,
98		Escape,
99		Hex,
100		Octal,
101		} = .Start;
102		var i: usize = 0;
103		var count: u8 = 0;
104		var num: u8 = 0;
105		for (tok.bytes) \|c\| {
106		switch (state) {
107		.Escape => {
108		switch (c) {
109		'n', 'r', 't', '\\', '\'', '\"' => {
110		bytes[i] = c;
111		},
112		'0'...'7' => {
113		count += 1;
114		num += c - '0';
115		state = .Octal;
116		bytes[i] = 'x';
117		},
118		'x' => {
119		state = .Hex;
120		bytes[i] = 'x';
121		},
122		'a' => {
123		bytes[i] = 'x';
124		i += 1;
125		bytes[i] = '0';
126		i += 1;
127		bytes[i] = '7';
128		},
129		'b' => {
130		bytes[i] = 'x';
131		i += 1;
132		bytes[i] = '0';
133		i += 1;
134		bytes[i] = '8';
135		},
136		'f' => {
137		bytes[i] = 'x';
138		i += 1;
139		bytes[i] = '0';
140		i += 1;
141		bytes[i] = 'C';
142		},
143		'v' => {
144		bytes[i] = 'x';
145		i += 1;
146		bytes[i] = '0';
147		i += 1;
148		bytes[i] = 'B';
149		},
150		'?' => {
151		i -= 1;
152		bytes[i] = '?';
153		},
154		'u', 'U' => {
155		try failDecl(ctx, loc, name, "macro tokenizing failed: TODO unicode escape sequences", .{});
156		return error.TokenizingFailed;
157		},
158		else => {
159		try failDecl(ctx, loc, name, "macro tokenizing failed: unknown escape sequence", .{});
160		return error.TokenizingFailed;
161		},
162		}
163		i += 1;
164		if (state == .Escape)
165		state = .Start;
166		},
167		.Start => {
168		if (c == '\\') {
169		state = .Escape;
170		}
171		bytes[i] = c;
172		i += 1;
173		},
174		.Hex => {
175		switch (c) {
176		'0'...'9' => {
177		num = std.math.mul(u8, num, 16) catch {
178		try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
179		return error.TokenizingFailed;
180		};
181		num += c - '0';
182		},
183		'a'...'f' => {
184		num = std.math.mul(u8, num, 16) catch {
185		try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
186		return error.TokenizingFailed;
187		};
188		num += c - 'a' + 10;
189		},
190		'A'...'F' => {
191		num = std.math.mul(u8, num, 16) catch {
192		try failDecl(ctx, loc, name, "macro tokenizing failed: hex literal overflowed", .{});
193		return error.TokenizingFailed;
194		};
195		num += c - 'A' + 10;
196		},
197		else => {
198		i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
199		num = 0;
200		if (c == '\\')
201		state = .Escape
202		else
203		state = .Start;
204		bytes[i] = c;
205		i += 1;
206		},
207		}
208		},
209		.Octal => {
210		const accept_digit = switch (c) {
211		// The maximum length of a octal literal is 3 digits
212		'0'...'7' => count < 3,
213		else => false,
214		};
215
216		if (accept_digit) {
217		count += 1;
218		num = std.math.mul(u8, num, 8) catch {
219		try failDecl(ctx, loc, name, "macro tokenizing failed: octal literal overflowed", .{});
220		return error.TokenizingFailed;
221		};
222		num += c - '0';
223		} else {
224		i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
225		num = 0;
226		count = 0;
227		if (c == '\\')
228		state = .Escape
229		else
230		state = .Start;
231		bytes[i] = c;
232		i += 1;
233		}
234		},
235		}
236		}
237		if (state == .Hex or state == .Octal)
238		i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
239		return CToken{
240		.id = tok.id,
241		.bytes = bytes[0..i],
242		};
243		}
244
245		fn next(ctx: Context, loc: ZigClangSourceLocation, name: []const u8, chars: [:0]const u8, i: *usize) !CToken {
246		var state: enum {
247		Start,
248		SawLt,
249		SawGt,
250		SawPlus,
251		SawMinus,
252		SawAmpersand,
253		SawPipe,
254		SawBang,
255		SawEq,
256		CharLit,
257		OpenComment,
258		Comment,
259		CommentStar,
260		Backslash,
261		String,
262		Identifier,
263		Decimal,
264		Octal,
265		SawZero,
266		Hex,
267		Bin,
268		Float,
269		ExpSign,
270		FloatExp,
271		FloatExpFirst,
272		NumLitIntSuffixU,
273		NumLitIntSuffixL,
274		NumLitIntSuffixLL,
275		NumLitIntSuffixUL,
276		Done,
277		} = .Start;
278
279		var result = CToken{
280		.bytes = "",
281		.id = .Eof,
282		};
283		var begin_index: usize = 0;
284		var digits: u8 = 0;
285		var pre_escape = state;
286
287		while (true) {
288		const c = chars[i.*];
289		if (c == 0) {
290		switch (state) {
291		.Identifier,
292		.Decimal,
293		.Hex,
294		.Bin,
295		.Octal,
296		.SawZero,
297		.Float,
298		.FloatExp,
299		=> {
300		result.bytes = chars[begin_index..i.*];
301		return result;
302		},
303		.Start,
304		.SawMinus,
305		.Done,
306		.NumLitIntSuffixU,
307		.NumLitIntSuffixL,
308		.NumLitIntSuffixUL,
309		.NumLitIntSuffixLL,
310		.SawLt,
311		.SawGt,
312		.SawPlus,
313		.SawAmpersand,
314		.SawPipe,
315		.SawBang,
316		.SawEq,
317		=> {
318		return result;
319		},
320		.CharLit,
321		.OpenComment,
322		.Comment,
323		.CommentStar,
324		.Backslash,
325		.String,
326		.ExpSign,
327		.FloatExpFirst,
328		=> {
329		try failDecl(ctx, loc, name, "macro tokenizing failed: unexpected EOF", .{});
330		return error.TokenizingFailed;
331		},
332		}
333		}
334		switch (state) {
335		.Start => {
336		switch (c) {
337		' ', '\t', '\x0B', '\x0C' => {},
338		'\'' => {
339		state = .CharLit;
340		result.id = .CharLit;
341		begin_index = i.*;
342		},
343		'\"' => {
344		state = .String;
345		result.id = .StrLit;
346		begin_index = i.*;
347		},
348		'/' => {
349		state = .OpenComment;
350		},
351		'\\' => {
352		state = .Backslash;
353		},
354		'\n', '\r' => {
355		return result;
356		},
357		'a'...'z', 'A'...'Z', '_' => {
358		state = .Identifier;
359		result.id = .Identifier;
360		begin_index = i.*;
361		},
362		'1'...'9' => {
363		state = .Decimal;
364		result.id = .NumLitInt;
365		begin_index = i.*;
366		},
367		'0' => {
368		state = .SawZero;
369		result.id = .NumLitInt;
370		begin_index = i.*;
371		},
372		'.' => {
373		result.id = .Dot;
374		state = .Done;
375		},
376		'<' => {
377		result.id = .Lt;
378		state = .SawLt;
379		},
380		'>' => {
381		result.id = .Gt;
382		state = .SawGt;
383		},
384		'(' => {
385		result.id = .LParen;
386		state = .Done;
387		},
388		')' => {
389		result.id = .RParen;
390		state = .Done;
391		},
392		'*' => {
393		result.id = .Asterisk;
394		state = .Done;
395		},
396		'+' => {
397		result.id = .Plus;
398		state = .SawPlus;
399		},
400		'-' => {
401		result.id = .Minus;
402		state = .SawMinus;
403		},
404		'!' => {
405		result.id = .Bang;
406		state = .SawBang;
407		},
408		'~' => {
409		result.id = .Tilde;
410		state = .Done;
411		},
412		'=' => {
413		result.id = .Assign;
414		state = .SawEq;
415		},
416		',' => {
417		result.id = .Comma;
418		state = .Done;
419		},
420		'[' => {
421		result.id = .LBrace;
422		state = .Done;
423		},
424		']' => {
425		result.id = .RBrace;
426		state = .Done;
427		},
428		'\|' => {
429		result.id = .Pipe;
430		state = .SawPipe;
431		},
432		'&' => {
433		result.id = .Ampersand;
434		state = .SawAmpersand;
435		},
436		'?' => {
437		result.id = .QuestionMark;
438		state = .Done;
439		},
440		':' => {
441		result.id = .Colon;
442		state = .Done;
443		},
444		else => {
445		try failDecl(ctx, loc, name, "macro tokenizing failed: unexpected character '{c}'", .{c});
446		return error.TokenizingFailed;
447		},
448		}
449		},
450		.Done => return result,
451		.SawMinus => {
452		switch (c) {
453		'>' => {
454		result.id = .Arrow;
455		state = .Done;
456		},
457		'-' => {
458		result.id = .Decrement;
459		state = .Done;
460		},
461		else => return result,
462		}
463		},
464		.SawPlus => {
465		switch (c) {
466		'+' => {
467		result.id = .Increment;
468		state = .Done;
469		},
470		else => return result,
471		}
472		},
473		.SawLt => {
474		switch (c) {
475		'<' => {
476		result.id = .Shl;
477		state = .Done;
478		},
479		'=' => {
480		result.id = .Lte;
481		state = .Done;
482		},
483		else => return result,
484		}
485		},
486		.SawGt => {
487		switch (c) {
488		'>' => {
489		result.id = .Shr;
490		state = .Done;
491		},
492		'=' => {
493		result.id = .Gte;
494		state = .Done;
495		},
496		else => return result,
497		}
498		},
499		.SawPipe => {
500		switch (c) {
501		'\|' => {
502		result.id = .Or;
503		state = .Done;
504		},
505		else => return result,
506		}
507		},
508		.SawAmpersand => {
509		switch (c) {
510		'&' => {
511		result.id = .And;
512		state = .Done;
513		},
514		else => return result,
515		}
516		},
517		.SawBang => {
518		switch (c) {
519		'=' => {
520		result.id = .Ne;
521		state = .Done;
522		},
523		else => return result,
524		}
525		},
526		.SawEq => {
527		switch (c) {
528		'=' => {
529		result.id = .Eq;
530		state = .Done;
531		},
532		else => return result,
533		}
534		},
535		.Float => {
536		switch (c) {
537		'.', '0'...'9' => {},
538		'e', 'E' => {
539		state = .ExpSign;
540		},
541		'f',
542		'F',
543		=> {
544		result.num_lit_suffix = .F;
545		result.bytes = chars[begin_index..i.*];
546		state = .Done;
547		},
548		'l', 'L' => {
549		result.num_lit_suffix = .L;
550		result.bytes = chars[begin_index..i.*];
551		state = .Done;
552		},
553		else => {
554		result.bytes = chars[begin_index..i.*];
555		return result;
556		},
557		}
558		},
559		.ExpSign => {
560		switch (c) {
561		'+', '-' => {
562		state = .FloatExpFirst;
563		},
564		'0'...'9' => {
565		state = .FloatExp;
566		},
567		else => {
568		try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit or '+' or '-'", .{});
569		return error.TokenizingFailed;
570		},
571		}
572		},
573		.FloatExpFirst => {
574		switch (c) {
575		'0'...'9' => {
576		state = .FloatExp;
577		},
578		else => {
579		try failDecl(ctx, loc, name, "macro tokenizing failed: expected a digit", .{});
580		return error.TokenizingFailed;
581		},
582		}
583		},
584		.FloatExp => {
585		switch (c) {
586		'0'...'9' => {},
587		'f', 'F' => {
588		result.num_lit_suffix = .F;
589		result.bytes = chars[begin_index..i.*];
590		state = .Done;
591		},
592		'l', 'L' => {
593		result.num_lit_suffix = .L;
594		result.bytes = chars[begin_index..i.*];
595		state = .Done;
596		},
597		else => {
598		result.bytes = chars[begin_index..i.*];
599		return result;
600		},
601		}
602		},
603		.Decimal => {
604		switch (c) {
605		'0'...'9' => {},
606		'\'' => {},
607		'u', 'U' => {
608		state = .NumLitIntSuffixU;
609		result.num_lit_suffix = .U;
610		result.bytes = chars[begin_index..i.*];
611		},
612		'l', 'L' => {
613		state = .NumLitIntSuffixL;
614		result.num_lit_suffix = .L;
615		result.bytes = chars[begin_index..i.*];
616		},
617		'.' => {
618		result.id = .NumLitFloat;
619		state = .Float;
620		},
621		else => {
622		result.bytes = chars[begin_index..i.*];
623		return result;
624		},
625		}
626		},
627		.SawZero => {
628		switch (c) {
629		'x', 'X' => {
630		state = .Hex;
631		},
632		'b', 'B' => {
633		state = .Bin;
634		},
635		'.' => {
636		state = .Float;
637		result.id = .NumLitFloat;
638		},
639		'u', 'U' => {
640		state = .NumLitIntSuffixU;
641		result.num_lit_suffix = .U;
642		result.bytes = chars[begin_index..i.*];
643		},
644		'l', 'L' => {
645		state = .NumLitIntSuffixL;
646		result.num_lit_suffix = .L;
647		result.bytes = chars[begin_index..i.*];
648		},
649		else => {
650		i.* -= 1;
651		state = .Octal;
652		},
653		}
654		},
655		.Octal => {
656		switch (c) {
657		'0'...'7' => {},
658		'8', '9' => {
659		try failDecl(ctx, loc, name, "macro tokenizing failed: invalid digit '{c}' in octal number", .{c});
660		return error.TokenizingFailed;
661		},
662		'u', 'U' => {
663		state = .NumLitIntSuffixU;
664		result.num_lit_suffix = .U;
665		result.bytes = chars[begin_index..i.*];
666		},
667		'l', 'L' => {
668		state = .NumLitIntSuffixL;
669		result.num_lit_suffix = .L;
670		result.bytes = chars[begin_index..i.*];
671		},
672		else => {
673		result.bytes = chars[begin_index..i.*];
674		return result;
675		},
676		}
677		},
678		.Hex => {
679		switch (c) {
680		'0'...'9', 'a'...'f', 'A'...'F' => {},
681		'u', 'U' => {
682		// marks the number literal as unsigned
683		state = .NumLitIntSuffixU;
684		result.num_lit_suffix = .U;
685		result.bytes = chars[begin_index..i.*];
686		},
687		'l', 'L' => {
688		// marks the number literal as long
689		state = .NumLitIntSuffixL;
690		result.num_lit_suffix = .L;
691		result.bytes = chars[begin_index..i.*];
692		},
693		else => {
694		result.bytes = chars[begin_index..i.*];
695		return result;
696		},
697		}
698		},
699		.Bin => {
700		switch (c) {
701		'0'...'1' => {},
702		'2'...'9' => {
703		try failDecl(ctx, loc, name, "macro tokenizing failed: invalid digit '{c}' in binary number", .{c});
704		return error.TokenizingFailed;
705		},
706		'u', 'U' => {
707		// marks the number literal as unsigned
708		state = .NumLitIntSuffixU;
709		result.num_lit_suffix = .U;
710		result.bytes = chars[begin_index..i.*];
711		},
712		'l', 'L' => {
713		// marks the number literal as long
714		state = .NumLitIntSuffixL;
715		result.num_lit_suffix = .L;
716		result.bytes = chars[begin_index..i.*];
717		},
718		else => {
719		result.bytes = chars[begin_index..i.*];
720		return result;
721		},
722		}
723		},
724		.NumLitIntSuffixU => {
725		switch (c) {
726		'l', 'L' => {
727		result.num_lit_suffix = .LU;
728		state = .NumLitIntSuffixUL;
729		},
730		else => {
731		return result;
732		},
733		}
734		},
735		.NumLitIntSuffixL => {
736		switch (c) {
737		'l', 'L' => {
738		result.num_lit_suffix = .LL;
739		state = .NumLitIntSuffixLL;
740		},
741		'u', 'U' => {
742		result.num_lit_suffix = .LU;
743		state = .Done;
744		},
745		else => {
746		return result;
747		},
748		}
749		},
750		.NumLitIntSuffixLL => {
751		switch (c) {
752		'u', 'U' => {
753		result.num_lit_suffix = .LLU;
754		state = .Done;
755		},
756		else => {
757		return result;
758		},
759		}
760		},
761		.NumLitIntSuffixUL => {
762		switch (c) {
763		'l', 'L' => {
764		result.num_lit_suffix = .LLU;
765		state = .Done;
766		},
767		else => {
768		return result;
769		},
770		}
771		},
772		.Identifier => {
773		switch (c) {
774		'_', 'a'...'z', 'A'...'Z', '0'...'9' => {},
775		else => {
776		result.bytes = chars[begin_index..i.*];
777		return result;
778		},
779		}
780		},
781		.String => {
782		switch (c) {
783		'\"' => {
784		result.bytes = chars[begin_index .. i.* + 1];
785		state = .Done;
786		},
787		else => {},
788		}
789		},
790		.CharLit => {
791		switch (c) {
792		'\'' => {
793		result.bytes = chars[begin_index .. i.* + 1];
794		state = .Done;
795		},
796		else => {},
797		}
798		},
799		.OpenComment => {
800		switch (c) {
801		'/' => {
802		return result;
803		},
804		'*' => {
805		state = .Comment;
806		},
807		else => {
808		result.id = .Slash;
809		state = .Done;
810		},
811		}
812		},
813		.Comment => {
814		switch (c) {
815		'*' => {
816		state = .CommentStar;
817		},
818		else => {},
819		}
820		},
821		.CommentStar => {
822		switch (c) {
823		'/' => {
824		state = .Start;
825		},
826		else => {
827		state = .Comment;
828		},
829		}
830		},
831		.Backslash => {
832		switch (c) {
833		' ', '\t', '\x0B', '\x0C' => {},
834		'\n', '\r' => {
835		state = .Start;
836		},
837		else => {
838		try failDecl(ctx, loc, name, "macro tokenizing failed: expected whitespace", .{});
839		return error.TokenizingFailed;
840		},
841		}
842		},
843		}
844		i.* += 1;
845		}
846		unreachable;
847		}
848
849		fn expectTokens(tl: TokenList, src: [:0]const u8, expected: []CToken) void {
850		// these can be undefined since they are only used for error reporting
851		tokenizeCMacro(undefined, undefined, undefined, tl, src) catch unreachable;
852		var it = tl.iterator(0);
853		for (expected) \|t\| {
854		var tok = it.next().?;
855		std.testing.expectEqual(t.id, tok.id);
856		if (t.bytes.len > 0) {
857		//std.debug.warn(" {} = {}\n", .{tok.bytes, t.bytes});
858		std.testing.expectEqualSlices(u8, tok.bytes, t.bytes);
859		}
860		if (t.num_lit_suffix != .None) {
861		std.testing.expectEqual(t.num_lit_suffix, tok.num_lit_suffix);
862		}
863		}
864		std.testing.expect(it.next() == null);
865		tl.shrink(0);
866		}
867
868		test "tokenize macro" {
869		var tl = TokenList.init(std.testing.allocator);
870		defer tl.deinit();
871
872		expectTokens(&tl, "TEST(0\n", &[_]CToken{
873		.{ .id = .Identifier, .bytes = "TEST" },
874		.{ .id = .Fn },
875		.{ .id = .LParen },
876		.{ .id = .NumLitInt, .bytes = "0" },
877		.{ .id = .Eof },
878		});
879
880		expectTokens(&tl, "__FLT_MIN_10_EXP__ -37\n", &[_]CToken{
881		.{ .id = .Identifier, .bytes = "__FLT_MIN_10_EXP__" },
882		.{ .id = .Minus },
883		.{ .id = .NumLitInt, .bytes = "37" },
884		.{ .id = .Eof },
885		});
886
887		expectTokens(&tl, "__llvm__ 1\n#define", &[_]CToken{
888		.{ .id = .Identifier, .bytes = "__llvm__" },
889		.{ .id = .NumLitInt, .bytes = "1" },
890		.{ .id = .Eof },
891		});
892
893		expectTokens(&tl, "TEST 2", &[_]CToken{
894		.{ .id = .Identifier, .bytes = "TEST" },
895		.{ .id = .NumLitInt, .bytes = "2" },
896		.{ .id = .Eof },
897		});
898
899		expectTokens(&tl, "FOO 0ull", &[_]CToken{
900		.{ .id = .Identifier, .bytes = "FOO" },
901		.{ .id = .NumLitInt, .bytes = "0", .num_lit_suffix = .LLU },
902		.{ .id = .Eof },
903		});
904		}
905
906		test "tokenize macro ops" {
907		var tl = TokenList.init(std.testing.allocator);
908		defer tl.deinit();
909
910		expectTokens(&tl, "ADD A + B", &[_]CToken{
911		.{ .id = .Identifier, .bytes = "ADD" },
912		.{ .id = .Identifier, .bytes = "A" },
913		.{ .id = .Plus },
914		.{ .id = .Identifier, .bytes = "B" },
915		.{ .id = .Eof },
916		});
917
918		expectTokens(&tl, "ADD (A) + B", &[_]CToken{
919		.{ .id = .Identifier, .bytes = "ADD" },
920		.{ .id = .LParen },
921		.{ .id = .Identifier, .bytes = "A" },
922		.{ .id = .RParen },
923		.{ .id = .Plus },
924		.{ .id = .Identifier, .bytes = "B" },
925		.{ .id = .Eof },
926		});
927
928		expectTokens(&tl, "ADD (A) + B", &[_]CToken{
929		.{ .id = .Identifier, .bytes = "ADD" },
930		.{ .id = .LParen },
931		.{ .id = .Identifier, .bytes = "A" },
932		.{ .id = .RParen },
933		.{ .id = .Plus },
934		.{ .id = .Identifier, .bytes = "B" },
935		.{ .id = .Eof },
936		});
937		}
938
939		test "escape sequences" {
940		var buf: [1024]u8 = undefined;
941		var alloc = std.heap.FixedBufferAllocator.init(buf[0..]);
942		const a = &alloc.allocator;
943		// these can be undefined since they are only used for error reporting
944		expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
945		.id = .StrLit,
946		.bytes = "\\x0077",
947		})).bytes, "\\x77"));
948		expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
949		.id = .StrLit,
950		.bytes = "\\24500",
951		})).bytes, "\\xa500"));
952		expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
953		.id = .StrLit,
954		.bytes = "\\x0077 abc",
955		})).bytes, "\\x77 abc"));
956		expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
957		.id = .StrLit,
958		.bytes = "\\045abc",
959		})).bytes, "\\x25abc"));
960
961		expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
962		.id = .CharLit,
963		.bytes = "\\0",
964		})).bytes, "\\x00"));
965		expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
966		.id = .CharLit,
967		.bytes = "\\00",
968		})).bytes, "\\x00"));
969		expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
970		.id = .CharLit,
971		.bytes = "\\000\\001",
972		})).bytes, "\\x00\\x01"));
973		expect(std.mem.eql(u8, (try zigifyEscapeSequences(undefined, undefined, undefined, a, .{
974		.id = .CharLit,
975		.bytes = "\\000abc",
976		})).bytes, "\\x00abc"));
977		}

src-self-hosted/translate_c.zig+320-98

...	...	@@ -6,8 +6,9 @@ const assert = std.debug.assert;
6	6	const ast = std.zig.ast;
7	7	const Token = std.zig.Token;
8	8	usingnamespace @import("clang.zig");
9		const ctok = @import("c_tokenizer.zig");
10		const CToken = ctok.CToken;
	9	const ctok = std.c.tokenizer;
	10	const CToken = std.c.Token;
	11	const CTokenList = std.c.tokenizer.Source.TokenList;
11	12	const mem = std.mem;
12	13	const math = std.math;
13	14
...	...	@@ -4810,6 +4811,15 @@ fn transCreateNodeIdentifier(c: Context, name: []const u8) !ast.Node {
4810	4811	return &identifier.base;
4811	4812	}
4812	4813
	4814	fn transCreateNodeTypeIdentifier(c: Context, name: []const u8) !ast.Node {
	4815	const token_index = try appendTokenFmt(c, .Identifier, "{}", .{name});
	4816	const identifier = try c.a().create(ast.Node.Identifier);
	4817	identifier.* = .{
	4818	.token = token_index,
	4819	};
	4820	return &identifier.base;
	4821	}
	4822
4813	4823	pub fn freeErrors(errors: []ClangErrMsg) void {
4814	4824	ZigClangErrorMsg_delete(errors.ptr, errors.len);
4815	4825	}
...	...	@@ -4818,7 +4828,7 @@ fn transPreprocessorEntities(c: Context, unit: ZigClangASTUnit) Error!void {
4818	4828	// TODO if we see #undef, delete it from the table
4819	4829	var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit);
4820	4830	const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit);
4821		var tok_list = ctok.TokenList.init(c.a());
	4831	var tok_list = CTokenList.init(c.a());
4822	4832	const scope = c.global_scope;
4823	4833
4824	4834	while (it.I != it_end.I) : (it.I += 1) {
...	...	@@ -4839,42 +4849,59 @@ fn transPreprocessorEntities(c: Context, unit: ZigClangASTUnit) Error!void {
4839	4849	}
4840	4850
4841	4851	const begin_c = ZigClangSourceManager_getCharacterData(c.source_manager, begin_loc);
4842		ctok.tokenizeCMacro(c, begin_loc, mangled_name, &tok_list, begin_c) catch \|err\| switch (err) {
4843		error.OutOfMemory => \|e\| return e,
4844		else => {
4845		continue;
	4852	const slice = begin_c[0..mem.len(u8, begin_c)];
	4853
	4854	tok_list.shrink(0);
	4855	var tokenizer = std.c.Tokenizer{
	4856	.source = &std.c.tokenizer.Source{
	4857	.buffer = slice,
	4858	.file_name = undefined,
	4859	.tokens = undefined,
4846	4860	},
4847	4861	};
	4862	while (true) {
	4863	const tok = tokenizer.next();
	4864	switch (tok.id) {
	4865	.Nl, .Eof => {
	4866	try tok_list.push(tok);
	4867	break;
	4868	},
	4869	.LineComment, .MultiLineComment => continue,
	4870	else => {},
	4871	}
	4872	try tok_list.push(tok);
	4873	}
4848	4874
4849	4875	var tok_it = tok_list.iterator(0);
4850	4876	const first_tok = tok_it.next().?;
4851		assert(first_tok.id == .Identifier and mem.eql(u8, first_tok.bytes, name));
	4877	assert(first_tok.id == .Identifier and mem.eql(u8, slice[first_tok.start..first_tok.end], name));
	4878
	4879	var macro_fn = false;
4852	4880	const next = tok_it.peek().?;
4853	4881	switch (next.id) {
4854	4882	.Identifier => {
4855	4883	// if it equals itself, ignore. for example, from stdio.h:
4856	4884	// #define stdin stdin
4857		if (mem.eql(u8, name, next.bytes)) {
	4885	if (mem.eql(u8, name, slice[next.start..next.end])) {
4858	4886	continue;
4859	4887	}
4860	4888	},
4861		.Eof => {
	4889	.Nl, .Eof => {
4862	4890	// this means it is a macro without a value
4863	4891	// we don't care about such things
4864	4892	continue;
4865	4893	},
	4894	.LParen => {
	4895	// if the name is immediately followed by a '(' then it is a function
	4896	macro_fn = first_tok.end == next.start;
	4897	},
4866	4898	else => {},
4867	4899	}
4868	4900
4869		const macro_fn = if (tok_it.peek().?.id == .Fn) blk: {
4870		_ = tok_it.next();
4871		break :blk true;
4872		} else false;
4873
4874	4901	(if (macro_fn)
4875		transMacroFnDefine(c, &tok_it, mangled_name, begin_loc)
	4902	transMacroFnDefine(c, &tok_it, slice, mangled_name, begin_loc)
4876	4903	else
4877		transMacroDefine(c, &tok_it, mangled_name, begin_loc)) catch \|err\| switch (err) {
	4904	transMacroDefine(c, &tok_it, slice, mangled_name, begin_loc)) catch \|err\| switch (err) {
4878	4905	error.ParseError => continue,
4879	4906	error.OutOfMemory => \|e\| return e,
4880	4907	};
...	...	@@ -4884,15 +4911,15 @@ fn transPreprocessorEntities(c: Context, unit: ZigClangASTUnit) Error!void {
4884	4911	}
4885	4912	}
4886	4913
4887		fn transMacroDefine(c: Context, it: ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
	4914	fn transMacroDefine(c: Context, it: CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
4888	4915	const scope = &c.global_scope.base;
4889	4916
4890	4917	const node = try transCreateNodeVarDecl(c, true, true, name);
4891	4918	node.eq_token = try appendToken(c, .Equal, "=");
4892	4919
4893		node.init_node = try parseCExpr(c, it, source_loc, scope);
	4920	node.init_node = try parseCExpr(c, it, source, source_loc, scope);
4894	4921	const last = it.next().?;
4895		if (last.id != .Eof)
	4922	if (last.id != .Eof and last.id != .Nl)
4896	4923	return failDecl(
4897	4924	c,
4898	4925	source_loc,
...	...	@@ -4905,7 +4932,7 @@ fn transMacroDefine(c: Context, it: ctok.TokenList.Iterator, name: []const u8,
4905	4932	_ = try c.global_scope.macro_table.put(name, &node.base);
4906	4933	}
4907	4934
4908		fn transMacroFnDefine(c: Context, it: ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
	4935	fn transMacroFnDefine(c: Context, it: CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
4909	4936	const block_scope = try Scope.Block.init(c, &c.global_scope.base, null);
4910	4937	const scope = &block_scope.base;
4911	4938
...	...	@@ -4937,7 +4964,7 @@ fn transMacroFnDefine(c: Context, it: ctok.TokenList.Iterator, name: []const u
4937	4964	);
4938	4965	}
4939	4966
4940		const mangled_name = try block_scope.makeMangledName(c, param_tok.bytes);
	4967	const mangled_name = try block_scope.makeMangledName(c, source[param_tok.start..param_tok.end]);
4941	4968	const param_name_tok = try appendIdentifier(c, mangled_name);
4942	4969	_ = try appendToken(c, .Colon, ":");
4943	4970
...	...	@@ -5000,9 +5027,9 @@ fn transMacroFnDefine(c: Context, it: ctok.TokenList.Iterator, name: []const u
5000	5027	const block = try transCreateNodeBlock(c, null);
5001	5028
5002	5029	const return_expr = try transCreateNodeReturnExpr(c);
5003		const expr = try parseCExpr(c, it, source_loc, scope);
	5030	const expr = try parseCExpr(c, it, source, source_loc, scope);
5004	5031	const last = it.next().?;
5005		if (last.id != .Eof)
	5032	if (last.id != .Eof and last.id != .Nl)
5006	5033	return failDecl(
5007	5034	c,
5008	5035	source_loc,
...	...	@@ -5022,27 +5049,28 @@ fn transMacroFnDefine(c: Context, it: ctok.TokenList.Iterator, name: []const u
5022	5049
5023	5050	const ParseError = Error \|\| error{ParseError};
5024	5051
5025		fn parseCExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: Scope) ParseError!ast.Node {
5026		const node = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5052	fn parseCExpr(c: Context, it: CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: Scope) ParseError!ast.Node {
	5053	const node = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5027	5054	switch (it.next().?.id) {
5028	5055	.QuestionMark => {
5029	5056	// must come immediately after expr
5030	5057	_ = try appendToken(c, .RParen, ")");
5031	5058	const if_node = try transCreateNodeIf(c);
5032	5059	if_node.condition = node;
5033		if_node.body = try parseCPrimaryExpr(c, it, source_loc, scope);
	5060	if_node.body = try parseCPrimaryExpr(c, it, source, source_loc, scope);
5034	5061	if (it.next().?.id != .Colon) {
	5062	const first_tok = it.list.at(0);
5035	5063	try failDecl(
5036	5064	c,
5037	5065	source_loc,
5038		it.list.at(0).*.bytes,
	5066	source[first_tok.start..first_tok.end],
5039	5067	"unable to translate C expr: expected ':'",
5040	5068	.{},
5041	5069	);
5042	5070	return error.ParseError;
5043	5071	}
5044	5072	if_node.@"else" = try transCreateNodeElse(c);
5045		if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source_loc, scope);
	5073	if_node.@"else".?.body = try parseCPrimaryExpr(c, it, source, source_loc, scope);
5046	5074	return &if_node.base;
5047	5075	},
5048	5076	else => {
...	...	@@ -5052,30 +5080,30 @@ fn parseCExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: ZigClangSou
5052	5080	}
5053	5081	}
5054	5082
5055		fn parseCNumLit(c: Context, tok: CToken, source_loc: ZigClangSourceLocation) ParseError!*ast.Node {
5056		if (tok.id == .NumLitInt) {
5057		var lit_bytes = tok.bytes;
	5083	fn parseCNumLit(c: Context, tok: CToken, source: []const u8, source_loc: ZigClangSourceLocation) ParseError!*ast.Node {
	5084	var lit_bytes = source[tok.start..tok.end];
5058	5085
5059		if (tok.bytes.len > 2 and tok.bytes[0] == '0') {
5060		switch (tok.bytes[1]) {
	5086	if (tok.id == .IntegerLiteral) {
	5087	if (lit_bytes.len > 2 and lit_bytes[0] == '0') {
	5088	switch (lit_bytes[1]) {
5061	5089	'0'...'7' => {
5062	5090	// Octal
5063		lit_bytes = try std.fmt.allocPrint(c.a(), "0o{}", .{tok.bytes});
	5091	lit_bytes = try std.fmt.allocPrint(c.a(), "0o{}", .{lit_bytes});
5064	5092	},
5065	5093	'X' => {
5066	5094	// Hexadecimal with capital X, valid in C but not in Zig
5067		lit_bytes = try std.fmt.allocPrint(c.a(), "0x{}", .{tok.bytes[2..]});
	5095	lit_bytes = try std.fmt.allocPrint(c.a(), "0x{}", .{lit_bytes[2..]});
5068	5096	},
5069	5097	else => {},
5070	5098	}
5071	5099	}
5072	5100
5073		if (tok.num_lit_suffix == .None) {
	5101	if (tok.id.IntegerLiteral == .None) {
5074	5102	return transCreateNodeInt(c, lit_bytes);
5075	5103	}
5076	5104
5077	5105	const cast_node = try transCreateNodeBuiltinFnCall(c, "@as");
5078		try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.num_lit_suffix) {
	5106	try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.id.IntegerLiteral) {
5079	5107	.U => "c_uint",
5080	5108	.L => "c_long",
5081	5109	.LU => "c_ulong",
...	...	@@ -5083,55 +5111,233 @@ fn parseCNumLit(c: Context, tok: CToken, source_loc: ZigClangSourceLocation) P
5083	5111	.LLU => "c_ulonglong",
5084	5112	else => unreachable,
5085	5113	}));
	5114	lit_bytes = lit_bytes[0 .. lit_bytes.len - switch (tok.id.IntegerLiteral) {
	5115	.U, .L => @as(u8, 1),
	5116	.LU, .LL => 2,
	5117	.LLU => 3,
	5118	else => unreachable,
	5119	}];
5086	5120	_ = try appendToken(c, .Comma, ",");
5087	5121	try cast_node.params.push(try transCreateNodeInt(c, lit_bytes));
5088	5122	cast_node.rparen_token = try appendToken(c, .RParen, ")");
5089	5123	return &cast_node.base;
5090		} else if (tok.id == .NumLitFloat) {
5091		if (tok.num_lit_suffix == .None) {
5092		return transCreateNodeFloat(c, tok.bytes);
	5124	} else if (tok.id == .FloatLiteral) {
	5125	if (tok.id.FloatLiteral == .None) {
	5126	return transCreateNodeFloat(c, lit_bytes);
5093	5127	}
5094	5128	const cast_node = try transCreateNodeBuiltinFnCall(c, "@as");
5095		try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.num_lit_suffix) {
	5129	try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.id.FloatLiteral) {
5096	5130	.F => "f32",
5097		.L => "f64",
	5131	.L => "c_longdouble",
5098	5132	else => unreachable,
5099	5133	}));
5100	5134	_ = try appendToken(c, .Comma, ",");
5101		try cast_node.params.push(try transCreateNodeFloat(c, tok.bytes));
	5135	try cast_node.params.push(try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1]));
5102	5136	cast_node.rparen_token = try appendToken(c, .RParen, ")");
5103	5137	return &cast_node.base;
5104	5138	} else unreachable;
5105	5139	}
5106	5140
5107		fn parseCPrimaryExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: Scope) ParseError!ast.Node {
	5141	fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ![]const u8 {
	5142	var source = source_bytes;
	5143	for (source) \|c, i\| {
	5144	if (c == '\"' or c == '\'') {
	5145	source = source[i..];
	5146	break;
	5147	}
	5148	}
	5149	for (source) \|c\| {
	5150	if (c == '\\') {
	5151	break;
	5152	}
	5153	} else return source;
	5154	var bytes = try ctx.a().alloc(u8, source.len * 2);
	5155	var state: enum {
	5156	Start,
	5157	Escape,
	5158	Hex,
	5159	Octal,
	5160	} = .Start;
	5161	var i: usize = 0;
	5162	var count: u8 = 0;
	5163	var num: u8 = 0;
	5164	for (source) \|c\| {
	5165	switch (state) {
	5166	.Escape => {
	5167	switch (c) {
	5168	'n', 'r', 't', '\\', '\'', '\"' => {
	5169	bytes[i] = c;
	5170	},
	5171	'0'...'7' => {
	5172	count += 1;
	5173	num += c - '0';
	5174	state = .Octal;
	5175	bytes[i] = 'x';
	5176	},
	5177	'x' => {
	5178	state = .Hex;
	5179	bytes[i] = 'x';
	5180	},
	5181	'a' => {
	5182	bytes[i] = 'x';
	5183	i += 1;
	5184	bytes[i] = '0';
	5185	i += 1;
	5186	bytes[i] = '7';
	5187	},
	5188	'b' => {
	5189	bytes[i] = 'x';
	5190	i += 1;
	5191	bytes[i] = '0';
	5192	i += 1;
	5193	bytes[i] = '8';
	5194	},
	5195	'f' => {
	5196	bytes[i] = 'x';
	5197	i += 1;
	5198	bytes[i] = '0';
	5199	i += 1;
	5200	bytes[i] = 'C';
	5201	},
	5202	'v' => {
	5203	bytes[i] = 'x';
	5204	i += 1;
	5205	bytes[i] = '0';
	5206	i += 1;
	5207	bytes[i] = 'B';
	5208	},
	5209	'?' => {
	5210	i -= 1;
	5211	bytes[i] = '?';
	5212	},
	5213	'u', 'U' => {
	5214	try failDecl(ctx, source_loc, name, "macro tokenizing failed: TODO unicode escape sequences", .{});
	5215	return error.ParseError;
	5216	},
	5217	else => {
	5218	try failDecl(ctx, source_loc, name, "macro tokenizing failed: unknown escape sequence", .{});
	5219	return error.ParseError;
	5220	},
	5221	}
	5222	i += 1;
	5223	if (state == .Escape)
	5224	state = .Start;
	5225	},
	5226	.Start => {
	5227	if (c == '\\') {
	5228	state = .Escape;
	5229	}
	5230	bytes[i] = c;
	5231	i += 1;
	5232	},
	5233	.Hex => {
	5234	switch (c) {
	5235	'0'...'9' => {
	5236	num = std.math.mul(u8, num, 16) catch {
	5237	try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
	5238	return error.ParseError;
	5239	};
	5240	num += c - '0';
	5241	},
	5242	'a'...'f' => {
	5243	num = std.math.mul(u8, num, 16) catch {
	5244	try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
	5245	return error.ParseError;
	5246	};
	5247	num += c - 'a' + 10;
	5248	},
	5249	'A'...'F' => {
	5250	num = std.math.mul(u8, num, 16) catch {
	5251	try failDecl(ctx, source_loc, name, "macro tokenizing failed: hex literal overflowed", .{});
	5252	return error.ParseError;
	5253	};
	5254	num += c - 'A' + 10;
	5255	},
	5256	else => {
	5257	i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
	5258	num = 0;
	5259	if (c == '\\')
	5260	state = .Escape
	5261	else
	5262	state = .Start;
	5263	bytes[i] = c;
	5264	i += 1;
	5265	},
	5266	}
	5267	},
	5268	.Octal => {
	5269	const accept_digit = switch (c) {
	5270	// The maximum length of a octal literal is 3 digits
	5271	'0'...'7' => count < 3,
	5272	else => false,
	5273	};
	5274
	5275	if (accept_digit) {
	5276	count += 1;
	5277	num = std.math.mul(u8, num, 8) catch {
	5278	try failDecl(ctx, source_loc, name, "macro tokenizing failed: octal literal overflowed", .{});
	5279	return error.ParseError;
	5280	};
	5281	num += c - '0';
	5282	} else {
	5283	i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
	5284	num = 0;
	5285	count = 0;
	5286	if (c == '\\')
	5287	state = .Escape
	5288	else
	5289	state = .Start;
	5290	bytes[i] = c;
	5291	i += 1;
	5292	}
	5293	},
	5294	}
	5295	}
	5296	if (state == .Hex or state == .Octal)
	5297	i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{ .fill = '0', .width = 2 });
	5298	return bytes[0..i];
	5299	}
	5300
	5301	fn parseCPrimaryExpr(c: Context, it: CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: Scope) ParseError!ast.Node {
5108	5302	const tok = it.next().?;
5109	5303	switch (tok.id) {
5110		.CharLit => {
5111		const token = try appendToken(c, .CharLiteral, tok.bytes);
	5304	.CharLiteral => {
	5305	const first_tok = it.list.at(0);
	5306	const token = try appendToken(c, .CharLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc));
5112	5307	const node = try c.a().create(ast.Node.CharLiteral);
5113	5308	node.* = ast.Node.CharLiteral{
5114	5309	.token = token,
5115	5310	};
5116	5311	return &node.base;
5117	5312	},
5118		.StrLit => {
5119		const token = try appendToken(c, .StringLiteral, tok.bytes);
	5313	.StringLiteral => {
	5314	const first_tok = it.list.at(0);
	5315	const token = try appendToken(c, .StringLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc));
5120	5316	const node = try c.a().create(ast.Node.StringLiteral);
5121	5317	node.* = ast.Node.StringLiteral{
5122	5318	.token = token,
5123	5319	};
5124	5320	return &node.base;
5125	5321	},
5126		.NumLitInt, .NumLitFloat => {
5127		return parseCNumLit(c, tok, source_loc);
	5322	.IntegerLiteral, .FloatLiteral => {
	5323	return parseCNumLit(c, tok, source, source_loc);
5128	5324	},
	5325	// eventually this will be replaced by std.c.parse which will handle these correctly
	5326	.Keyword_void => return transCreateNodeTypeIdentifier(c, "c_void"),
	5327	.Keyword_bool => return transCreateNodeTypeIdentifier(c, "bool"),
	5328	.Keyword_double => return transCreateNodeTypeIdentifier(c, "f64"),
	5329	.Keyword_long => return transCreateNodeTypeIdentifier(c, "c_long"),
	5330	.Keyword_int => return transCreateNodeTypeIdentifier(c, "c_int"),
	5331	.Keyword_float => return transCreateNodeTypeIdentifier(c, "f32"),
	5332	.Keyword_short => return transCreateNodeTypeIdentifier(c, "c_short"),
	5333	.Keyword_char => return transCreateNodeTypeIdentifier(c, "c_char"),
	5334	.Keyword_unsigned => return transCreateNodeTypeIdentifier(c, "c_uint"),
5129	5335	.Identifier => {
5130		const mangled_name = scope.getAlias(tok.bytes);
	5336	const mangled_name = scope.getAlias(source[tok.start..tok.end]);
5131	5337	return transCreateNodeIdentifier(c, mangled_name);
5132	5338	},
5133	5339	.LParen => {
5134		const inner_node = try parseCExpr(c, it, source_loc, scope);
	5340	const inner_node = try parseCExpr(c, it, source, source_loc, scope);
5135	5341
5136	5342	if (it.peek().?.id == .RParen) {
5137	5343	_ = it.next();
...	...	@@ -5144,13 +5350,14 @@ fn parseCPrimaryExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: ZigC
5144	5350	// hack to get zig fmt to render a comma in builtin calls
5145	5351	_ = try appendToken(c, .Comma, ",");
5146	5352
5147		const node_to_cast = try parseCExpr(c, it, source_loc, scope);
	5353	const node_to_cast = try parseCExpr(c, it, source, source_loc, scope);
5148	5354
5149	5355	if (it.next().?.id != .RParen) {
	5356	const first_tok = it.list.at(0);
5150	5357	try failDecl(
5151	5358	c,
5152	5359	source_loc,
5153		it.list.at(0).*.bytes,
	5360	source[first_tok.start..first_tok.end],
5154	5361	"unable to translate C expr: expected ')''",
5155	5362	.{},
5156	5363	);
...	...	@@ -5228,10 +5435,11 @@ fn parseCPrimaryExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: ZigC
5228	5435	return &if_1.base;
5229	5436	},
5230	5437	else => {
	5438	const first_tok = it.list.at(0);
5231	5439	try failDecl(
5232	5440	c,
5233	5441	source_loc,
5234		it.list.at(0).*.bytes,
	5442	source[first_tok.start..first_tok.end],
5235	5443	"unable to translate C expr: unexpected token {}",
5236	5444	.{tok.id},
5237	5445	);
...	...	@@ -5240,33 +5448,35 @@ fn parseCPrimaryExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: ZigC
5240	5448	}
5241	5449	}
5242	5450
5243		fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: Scope) ParseError!ast.Node {
5244		var node = try parseCPrimaryExpr(c, it, source_loc, scope);
	5451	fn parseCSuffixOpExpr(c: Context, it: CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: Scope) ParseError!ast.Node {
	5452	var node = try parseCPrimaryExpr(c, it, source, source_loc, scope);
5245	5453	while (true) {
5246	5454	const tok = it.next().?;
5247	5455	switch (tok.id) {
5248		.Dot => {
	5456	.Period => {
5249	5457	const name_tok = it.next().?;
5250	5458	if (name_tok.id != .Identifier) {
	5459	const first_tok = it.list.at(0);
5251	5460	try failDecl(
5252	5461	c,
5253	5462	source_loc,
5254		it.list.at(0).*.bytes,
	5463	source[first_tok.start..first_tok.end],
5255	5464	"unable to translate C expr: expected identifier",
5256	5465	.{},
5257	5466	);
5258	5467	return error.ParseError;
5259	5468	}
5260	5469
5261		node = try transCreateNodeFieldAccess(c, node, name_tok.bytes);
	5470	node = try transCreateNodeFieldAccess(c, node, source[name_tok.start..name_tok.end]);
5262	5471	},
5263	5472	.Arrow => {
5264	5473	const name_tok = it.next().?;
5265	5474	if (name_tok.id != .Identifier) {
	5475	const first_tok = it.list.at(0);
5266	5476	try failDecl(
5267	5477	c,
5268	5478	source_loc,
5269		it.list.at(0).*.bytes,
	5479	source[first_tok.start..first_tok.end],
5270	5480	"unable to translate C expr: expected identifier",
5271	5481	.{},
5272	5482	);
...	...	@@ -5274,7 +5484,7 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5274	5484	}
5275	5485
5276	5486	const deref = try transCreateNodePtrDeref(c, node);
5277		node = try transCreateNodeFieldAccess(c, deref, name_tok.bytes);
	5487	node = try transCreateNodeFieldAccess(c, deref, source[name_tok.start..name_tok.end]);
5278	5488	},
5279	5489	.Asterisk => {
5280	5490	if (it.peek().?.id == .RParen) {
...	...	@@ -5283,13 +5493,23 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5283	5493	// hack to get zig fmt to render a comma in builtin calls
5284	5494	_ = try appendToken(c, .Comma, ",");
5285	5495
5286		const ptr = try transCreateNodePtrType(c, false, false, .Identifier);
	5496	const ptr_kind = blk:{
	5497	// * token
	5498	_ = it.prev();
	5499	// last token of `node`
	5500	const prev_id = it.prev().?.id;
	5501	_ = it.next();
	5502	_ = it.next();
	5503	break :blk if (prev_id == .Keyword_void) .Asterisk else Token.Id.Identifier;
	5504	};
	5505
	5506	const ptr = try transCreateNodePtrType(c, false, false, ptr_kind);
5287	5507	ptr.rhs = node;
5288	5508	return &ptr.base;
5289	5509	} else {
5290	5510	// expr * expr
5291	5511	const op_token = try appendToken(c, .Asterisk, "*");
5292		const rhs = try parseCPrimaryExpr(c, it, source_loc, scope);
	5512	const rhs = try parseCPrimaryExpr(c, it, source, source_loc, scope);
5293	5513	const mul_node = try c.a().create(ast.Node.InfixOp);
5294	5514	mul_node.* = .{
5295	5515	.op_token = op_token,
...	...	@@ -5300,9 +5520,9 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5300	5520	node = &mul_node.base;
5301	5521	}
5302	5522	},
5303		.Shl => {
	5523	.AngleBracketAngleBracketLeft => {
5304	5524	const op_token = try appendToken(c, .AngleBracketAngleBracketLeft, "<<");
5305		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5525	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5306	5526	const bitshift_node = try c.a().create(ast.Node.InfixOp);
5307	5527	bitshift_node.* = .{
5308	5528	.op_token = op_token,
...	...	@@ -5312,9 +5532,9 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5312	5532	};
5313	5533	node = &bitshift_node.base;
5314	5534	},
5315		.Shr => {
	5535	.AngleBracketAngleBracketRight => {
5316	5536	const op_token = try appendToken(c, .AngleBracketAngleBracketRight, ">>");
5317		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5537	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5318	5538	const bitshift_node = try c.a().create(ast.Node.InfixOp);
5319	5539	bitshift_node.* = .{
5320	5540	.op_token = op_token,
...	...	@@ -5326,7 +5546,7 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5326	5546	},
5327	5547	.Pipe => {
5328	5548	const op_token = try appendToken(c, .Pipe, "\|");
5329		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5549	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5330	5550	const or_node = try c.a().create(ast.Node.InfixOp);
5331	5551	or_node.* = .{
5332	5552	.op_token = op_token,
...	...	@@ -5338,7 +5558,7 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5338	5558	},
5339	5559	.Ampersand => {
5340	5560	const op_token = try appendToken(c, .Ampersand, "&");
5341		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5561	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5342	5562	const bitand_node = try c.a().create(ast.Node.InfixOp);
5343	5563	bitand_node.* = .{
5344	5564	.op_token = op_token,
...	...	@@ -5350,7 +5570,7 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5350	5570	},
5351	5571	.Plus => {
5352	5572	const op_token = try appendToken(c, .Plus, "+");
5353		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5573	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5354	5574	const add_node = try c.a().create(ast.Node.InfixOp);
5355	5575	add_node.* = .{
5356	5576	.op_token = op_token,
...	...	@@ -5362,7 +5582,7 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5362	5582	},
5363	5583	.Minus => {
5364	5584	const op_token = try appendToken(c, .Minus, "-");
5365		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5585	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5366	5586	const sub_node = try c.a().create(ast.Node.InfixOp);
5367	5587	sub_node.* = .{
5368	5588	.op_token = op_token,
...	...	@@ -5372,9 +5592,9 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5372	5592	};
5373	5593	node = &sub_node.base;
5374	5594	},
5375		.And => {
	5595	.AmpersandAmpersand => {
5376	5596	const op_token = try appendToken(c, .Keyword_and, "and");
5377		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5597	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5378	5598	const and_node = try c.a().create(ast.Node.InfixOp);
5379	5599	and_node.* = .{
5380	5600	.op_token = op_token,
...	...	@@ -5384,9 +5604,9 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5384	5604	};
5385	5605	node = &and_node.base;
5386	5606	},
5387		.Or => {
	5607	.PipePipe => {
5388	5608	const op_token = try appendToken(c, .Keyword_or, "or");
5389		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5609	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5390	5610	const or_node = try c.a().create(ast.Node.InfixOp);
5391	5611	or_node.* = .{
5392	5612	.op_token = op_token,
...	...	@@ -5396,9 +5616,9 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5396	5616	};
5397	5617	node = &or_node.base;
5398	5618	},
5399		.Gt => {
	5619	.AngleBracketRight => {
5400	5620	const op_token = try appendToken(c, .AngleBracketRight, ">");
5401		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5621	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5402	5622	const and_node = try c.a().create(ast.Node.InfixOp);
5403	5623	and_node.* = .{
5404	5624	.op_token = op_token,
...	...	@@ -5408,9 +5628,9 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5408	5628	};
5409	5629	node = &and_node.base;
5410	5630	},
5411		.Gte => {
	5631	.AngleBracketRightEqual => {
5412	5632	const op_token = try appendToken(c, .AngleBracketRightEqual, ">=");
5413		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5633	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5414	5634	const and_node = try c.a().create(ast.Node.InfixOp);
5415	5635	and_node.* = .{
5416	5636	.op_token = op_token,
...	...	@@ -5420,9 +5640,9 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5420	5640	};
5421	5641	node = &and_node.base;
5422	5642	},
5423		.Lt => {
	5643	.AngleBracketLeft => {
5424	5644	const op_token = try appendToken(c, .AngleBracketLeft, "<");
5425		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5645	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5426	5646	const and_node = try c.a().create(ast.Node.InfixOp);
5427	5647	and_node.* = .{
5428	5648	.op_token = op_token,
...	...	@@ -5432,9 +5652,9 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5432	5652	};
5433	5653	node = &and_node.base;
5434	5654	},
5435		.Lte => {
	5655	.AngleBracketLeftEqual => {
5436	5656	const op_token = try appendToken(c, .AngleBracketLeftEqual, "<=");
5437		const rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5657	const rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5438	5658	const and_node = try c.a().create(ast.Node.InfixOp);
5439	5659	and_node.* = .{
5440	5660	.op_token = op_token,
...	...	@@ -5444,16 +5664,17 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5444	5664	};
5445	5665	node = &and_node.base;
5446	5666	},
5447		.LBrace => {
	5667	.LBracket => {
5448	5668	const arr_node = try transCreateNodeArrayAccess(c, node);
5449		arr_node.op.ArrayAccess = try parseCPrefixOpExpr(c, it, source_loc, scope);
5450		arr_node.rtoken = try appendToken(c, .RBrace, "]");
	5669	arr_node.op.ArrayAccess = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
	5670	arr_node.rtoken = try appendToken(c, .RBracket, "]");
5451	5671	node = &arr_node.base;
5452		if (it.next().?.id != .RBrace) {
	5672	if (it.next().?.id != .RBracket) {
	5673	const first_tok = it.list.at(0);
5453	5674	try failDecl(
5454	5675	c,
5455	5676	source_loc,
5456		it.list.at(0).*.bytes,
	5677	source[first_tok.start..first_tok.end],
5457	5678	"unable to translate C expr: expected ']'",
5458	5679	.{},
5459	5680	);
...	...	@@ -5463,7 +5684,7 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5463	5684	.LParen => {
5464	5685	const call_node = try transCreateNodeFnCall(c, node);
5465	5686	while (true) {
5466		const arg = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5687	const arg = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5467	5688	try call_node.op.Call.params.push(arg);
5468	5689	const next = it.next().?;
5469	5690	if (next.id == .Comma)
...	...	@@ -5471,10 +5692,11 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5471	5692	else if (next.id == .RParen)
5472	5693	break
5473	5694	else {
	5695	const first_tok = it.list.at(0);
5474	5696	try failDecl(
5475	5697	c,
5476	5698	source_loc,
5477		it.list.at(0).*.bytes,
	5699	source[first_tok.start..first_tok.end],
5478	5700	"unable to translate C expr: expected ',' or ')'",
5479	5701	.{},
5480	5702	);
...	...	@@ -5492,32 +5714,32 @@ fn parseCSuffixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: Zig
5492	5714	}
5493	5715	}
5494	5716
5495		fn parseCPrefixOpExpr(c: Context, it: ctok.TokenList.Iterator, source_loc: ZigClangSourceLocation, scope: Scope) ParseError!ast.Node {
	5717	fn parseCPrefixOpExpr(c: Context, it: CTokenList.Iterator, source: []const u8, source_loc: ZigClangSourceLocation, scope: Scope) ParseError!ast.Node {
5496	5718	const op_tok = it.next().?;
5497	5719
5498	5720	switch (op_tok.id) {
5499	5721	.Bang => {
5500	5722	const node = try transCreateNodePrefixOp(c, .BoolNot, .Bang, "!");
5501		node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5723	node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5502	5724	return &node.base;
5503	5725	},
5504	5726	.Minus => {
5505	5727	const node = try transCreateNodePrefixOp(c, .Negation, .Minus, "-");
5506		node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5728	node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5507	5729	return &node.base;
5508	5730	},
5509	5731	.Tilde => {
5510	5732	const node = try transCreateNodePrefixOp(c, .BitNot, .Tilde, "~");
5511		node.rhs = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5733	node.rhs = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5512	5734	return &node.base;
5513	5735	},
5514	5736	.Asterisk => {
5515		const prefix_op_expr = try parseCPrefixOpExpr(c, it, source_loc, scope);
	5737	const prefix_op_expr = try parseCPrefixOpExpr(c, it, source, source_loc, scope);
5516	5738	return try transCreateNodePtrDeref(c, prefix_op_expr);
5517	5739	},
5518	5740	else => {
5519	5741	_ = it.prev();
5520		return try parseCSuffixOpExpr(c, it, source_loc, scope);
	5742	return try parseCSuffixOpExpr(c, it, source, source_loc, scope);
5521	5743	},
5522	5744	}
5523	5745	}

test/translate_c.zig+24-2

...	...	@@ -618,6 +618,28 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
618	618	},
619	619	);
620	620
	621	cases.add("float suffixes",
	622	\\#define foo 3.14f
	623	\\#define bar 16.e-2l
	624	, &[_][]const u8{
	625	"pub const foo = @as(f32, 3.14);",
	626	"pub const bar = @as(c_longdouble, 16.e-2);",
	627	});
	628
	629	cases.add("comments",
	630	\\#define foo 1 //foo
	631	\\#define bar /* bar */ 2
	632	, &[_][]const u8{
	633	"pub const foo = 1;",
	634	"pub const bar = 2;",
	635	});
	636
	637	cases.add("string prefix",
	638	\\#define foo L"hello"
	639	, &[_][]const u8{
	640	"pub const foo = \"hello\";",
	641	});
	642
621	643	cases.add("null statements",
622	644	\\void foo(void) {
623	645	\\ ;;;;;
...	...	@@ -2508,8 +2530,8 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
2508	2530	cases.add("macro cast",
2509	2531	\\#define FOO(bar) baz((void *)(baz))
2510	2532	, &[_][]const u8{
2511		\\pub inline fn FOO(bar: var) @TypeOf(baz(if (@typeId(@TypeOf(baz)) == .Pointer) @ptrCast([c]void, baz) else if (@typeId(@TypeOf(baz)) == .Int) @intToPtr([c]void, baz) else @as([*c]void, baz))) {
2512		\\ return baz(if (@typeId(@TypeOf(baz)) == .Pointer) @ptrCast([c]void, baz) else if (@typeId(@TypeOf(baz)) == .Int) @intToPtr([c]void, baz) else @as([*c]void, baz));
	2533	\\pub inline fn FOO(bar: var) @TypeOf(baz(if (@typeId(@TypeOf(baz)) == .Pointer) @ptrCast(c_void, baz) else if (@typeId(@TypeOf(baz)) == .Int) @intToPtr(c_void, baz) else @as(*c_void, baz))) {
	2534	\\ return baz(if (@typeId(@TypeOf(baz)) == .Pointer) @ptrCast(c_void, baz) else if (@typeId(@TypeOf(baz)) == .Int) @intToPtr(c_void, baz) else @as(*c_void, baz));
2513	2535	\\}
2514	2536	});
2515	2537