authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-16 00:22:41+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-16 00:55:50+02:00
logab60c8e28fb89e33b094e457d477a19d1f015c62
treea89d7cd4b54f67fdc71080a0d0a195fb2a2dad6c
parent9f0e83a5710a85273f12b7e9ecf68c93e0f763e8
signature Commit is signed but in an unrecognized format.

c tokenizer escape sequences

4 files changed, 100 insertions(+), 20 deletions(-)

src-self-hosted/c_tokenizer.zig+84-5
......@@ -40,12 +40,15 @@ pub const CToken = struct {
4040 };
4141};
4242
43pub fn tokenizeCMacro(tl: *TokenList, chars: [*]const u8) !void {
43pub fn tokenizeCMacro(tl: *TokenList, chars: [*:0]const u8) !void {
4444 var index: usize = 0;
4545 var first = true;
4646 while (true) {
4747 const tok = try next(chars, &index);
48 try tl.push(tok);
48 if (tok.id == .StrLit or tok.id == .CharLit)
49 try tl.push(try zigifyEscapeSequences(tl.allocator, tok))
50 else
51 try tl.push(tok);
4952 if (tok.id == .Eof)
5053 return;
5154 if (first) {
......@@ -61,7 +64,83 @@ pub fn tokenizeCMacro(tl: *TokenList, chars: [*]const u8) !void {
6164 }
6265}
6366
64fn next(chars: [*]const u8, i: *usize) !CToken {
67fn zigifyEscapeSequences(allocator: *std.mem.Allocator, tok: CToken) !CToken {
68 for (tok.bytes) |c| {
69 if (c == '\\') {
70 break;
71 }
72 } else return tok;
73 var bytes = try allocator.alloc(u8, tok.bytes.len * 2);
74 var escape = false;
75 var i: usize = 0;
76 for (tok.bytes) |c| {
77 if (escape) {
78 switch (c) {
79 'n', 'r', 't', '\\', '\'', '\"', 'x' => {
80 bytes[i] = c;
81 },
82 'a' => {
83 bytes[i] = 'x';
84 i += 1;
85 bytes[i] = '0';
86 i += 1;
87 bytes[i] = '7';
88 },
89 'b' => {
90 bytes[i] = 'x';
91 i += 1;
92 bytes[i] = '0';
93 i += 1;
94 bytes[i] = '8';
95 },
96 'f' => {
97 bytes[i] = 'x';
98 i += 1;
99 bytes[i] = '0';
100 i += 1;
101 bytes[i] = 'C';
102 },
103 'v' => {
104 bytes[i] = 'x';
105 i += 1;
106 bytes[i] = '0';
107 i += 1;
108 bytes[i] = 'B';
109 },
110 '?' => {
111 i -= 1;
112 bytes[i] = '?';
113 },
114 'u', 'U' => {
115 // TODO unicode escape sequences
116 return error.TokenizingFailed;
117 },
118 '0'...'7' => {
119 // TODO octal escape sequences
120 return error.TokenizingFailed;
121 },
122 else => {
123 // unknown escape sequence
124 return error.TokenizingFailed;
125 },
126 }
127 i += 1;
128 escape = false;
129 } else {
130 if (c == '\\') {
131 escape = true;
132 }
133 bytes[i] = c;
134 i += 1;
135 }
136 }
137 return CToken{
138 .id = tok.id,
139 .bytes = bytes[0..i],
140 };
141}
142
143fn next(chars: [*:0]const u8, i: *usize) !CToken {
65144 var state: enum {
66145 Start,
67146 GotLt,
......@@ -462,7 +541,7 @@ fn next(chars: [*]const u8, i: *usize) !CToken {
462541 .String => { // TODO char escapes
463542 switch (c) {
464543 '\"' => {
465 result.bytes = chars[begin_index + 1 .. i.* - 1];
544 result.bytes = chars[begin_index..i.*];
466545 return result;
467546 },
468547 else => {},
......@@ -471,7 +550,7 @@ fn next(chars: [*]const u8, i: *usize) !CToken {
471550 .CharLit => {
472551 switch (c) {
473552 '\'' => {
474 result.bytes = chars[begin_index + 1 .. i.* - 1];
553 result.bytes = chars[begin_index..i.*];
475554 return result;
476555 },
477556 else => {},
src-self-hosted/clang.zig+1-1
......@@ -734,7 +734,7 @@ pub extern fn ZigClangSourceManager_getSpellingLoc(self: ?*const struct_ZigClang
734734pub extern fn ZigClangSourceManager_getFilename(self: *const struct_ZigClangSourceManager, SpellingLoc: struct_ZigClangSourceLocation) ?[*:0]const u8;
735735pub extern fn ZigClangSourceManager_getSpellingLineNumber(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) c_uint;
736736pub extern fn ZigClangSourceManager_getSpellingColumnNumber(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) c_uint;
737pub extern fn ZigClangSourceManager_getCharacterData(self: ?*const struct_ZigClangSourceManager, SL: struct_ZigClangSourceLocation) [*c]const u8;
737pub extern fn ZigClangSourceManager_getCharacterData(self: ?*const struct_ZigClangSourceManager, SL: struct_ZigClangSourceLocation) [*:0]const u8;
738738pub extern fn ZigClangASTContext_getPointerType(self: ?*const struct_ZigClangASTContext, T: struct_ZigClangQualType) struct_ZigClangQualType;
739739pub extern fn ZigClangASTUnit_getASTContext(self: ?*struct_ZigClangASTUnit) ?*struct_ZigClangASTContext;
740740pub extern fn ZigClangASTUnit_getSourceManager(self: *struct_ZigClangASTUnit) *struct_ZigClangSourceManager;
src-self-hosted/translate_c.zig+6-14
......@@ -2629,9 +2629,9 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
26292629 } else false;
26302630
26312631 (if (macro_fn)
2632 transMacroFnDefine(c, &tok_it, name, begin_c, begin_loc)
2632 transMacroFnDefine(c, &tok_it, name, begin_loc)
26332633 else
2634 transMacroDefine(c, &tok_it, name, begin_c, begin_loc)) catch |err| switch (err) {
2634 transMacroDefine(c, &tok_it, name, begin_loc)) catch |err| switch (err) {
26352635 error.UnsupportedTranslation,
26362636 error.ParseError,
26372637 => try failDecl(c, begin_loc, name, "unable to translate macro", .{}),
......@@ -2643,7 +2643,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
26432643 }
26442644}
26452645
2646fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, char_ptr: [*]const u8, source_loc: ZigClangSourceLocation) ParseError!void {
2646fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
26472647 const rp = makeRestorePoint(c);
26482648
26492649 const visib_tok = try appendToken(c, .Keyword_pub, "pub");
......@@ -2674,7 +2674,7 @@ fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8,
26742674 _ = try c.macro_table.put(name, &node.base);
26752675}
26762676
2677fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, char_ptr: [*]const u8, source_loc: ZigClangSourceLocation) ParseError!void {
2677fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
26782678 const rp = makeRestorePoint(c);
26792679 const pub_tok = try appendToken(c, .Keyword_pub, "pub");
26802680 const inline_tok = try appendToken(c, .Keyword_inline, "inline");
......@@ -2829,11 +2829,7 @@ fn parseCPrimaryExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc:
28292829 const tok = it.next().?;
28302830 switch (tok.id) {
28312831 .CharLit => {
2832 const buf = try rp.c.a().alloc(u8, tok.bytes.len + "''".len);
2833 buf[0] = '\'';
2834 writeEscapedString(buf[1..], tok.bytes);
2835 buf[buf.len - 1] = '\'';
2836 const token = try appendToken(rp.c, .CharLiteral, buf);
2832 const token = try appendToken(rp.c, .CharLiteral, tok.bytes);
28372833 const node = try rp.c.a().create(ast.Node.CharLiteral);
28382834 node.* = ast.Node.CharLiteral{
28392835 .token = token,
......@@ -2841,11 +2837,7 @@ fn parseCPrimaryExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc:
28412837 return &node.base;
28422838 },
28432839 .StrLit => {
2844 const buf = try rp.c.a().alloc(u8, tok.bytes.len + "\"\"".len);
2845 buf[0] = '"';
2846 writeEscapedString(buf[1..], tok.bytes);
2847 buf[buf.len - 1] = '"';
2848 const token = try appendToken(rp.c, .StringLiteral, buf);
2840 const token = try appendToken(rp.c, .StringLiteral, tok.bytes);
28492841 const node = try rp.c.a().create(ast.Node.StringLiteral);
28502842 node.* = ast.Node.StringLiteral{
28512843 .token = token,
test/translate_c.zig+9
......@@ -411,6 +411,15 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
411411 \\}
412412 });
413413
414 cases.add_2("macro escape sequences",
415 \\#define FOO "aoeu\xab derp"
416 \\#define FOO2 "aoeu\a derp"
417 , &[_][]const u8{
418 \\pub const FOO = "aoeu\xab derp";
419 ,
420 \\pub const FOO2 = "aoeu\x07 derp";
421 });
422
414423 /////////////// Cases for only stage1 which are TODO items for stage2 ////////////////
415424
416425 cases.add_both("typedef of function in struct field",