authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-16 00:22:41+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-16 00:55:50+02:00
logab60c8e28fb89e33b094e457d477a19d1f015c62
treea89d7cd4b54f67fdc71080a0d0a195fb2a2dad6c
parent9f0e83a5710a85273f12b7e9ecf68c93e0f763e8
signature Commit is signed but in an unrecognized format.

c tokenizer escape sequences

4 files changed, 100 insertions(+), 20 deletions(-)

src-self-hosted/c_tokenizer.zig+84-5
...@@ -40,12 +40,15 @@ pub const CToken = struct {...@@ -40,12 +40,15 @@ pub const CToken = struct {
40 };40 };
41};41};
4242
43pub fn tokenizeCMacro(tl: *TokenList, chars: [*]const u8) !void {43pub fn tokenizeCMacro(tl: *TokenList, chars: [*:0]const u8) !void {
44 var index: usize = 0;44 var index: usize = 0;
45 var first = true;45 var first = true;
46 while (true) {46 while (true) {
47 const tok = try next(chars, &index);47 const tok = try next(chars, &index);
48 try tl.push(tok);48 if (tok.id == .StrLit or tok.id == .CharLit)
49 try tl.push(try zigifyEscapeSequences(tl.allocator, tok))
50 else
51 try tl.push(tok);
49 if (tok.id == .Eof)52 if (tok.id == .Eof)
50 return;53 return;
51 if (first) {54 if (first) {
...@@ -61,7 +64,83 @@ pub fn tokenizeCMacro(tl: *TokenList, chars: [*]const u8) !void {...@@ -61,7 +64,83 @@ pub fn tokenizeCMacro(tl: *TokenList, chars: [*]const u8) !void {
61 }64 }
62}65}
6366
64fn next(chars: [*]const u8, i: *usize) !CToken {67fn zigifyEscapeSequences(allocator: *std.mem.Allocator, tok: CToken) !CToken {
68 for (tok.bytes) |c| {
69 if (c == '\\') {
70 break;
71 }
72 } else return tok;
73 var bytes = try allocator.alloc(u8, tok.bytes.len * 2);
74 var escape = false;
75 var i: usize = 0;
76 for (tok.bytes) |c| {
77 if (escape) {
78 switch (c) {
79 'n', 'r', 't', '\\', '\'', '\"', 'x' => {
80 bytes[i] = c;
81 },
82 'a' => {
83 bytes[i] = 'x';
84 i += 1;
85 bytes[i] = '0';
86 i += 1;
87 bytes[i] = '7';
88 },
89 'b' => {
90 bytes[i] = 'x';
91 i += 1;
92 bytes[i] = '0';
93 i += 1;
94 bytes[i] = '8';
95 },
96 'f' => {
97 bytes[i] = 'x';
98 i += 1;
99 bytes[i] = '0';
100 i += 1;
101 bytes[i] = 'C';
102 },
103 'v' => {
104 bytes[i] = 'x';
105 i += 1;
106 bytes[i] = '0';
107 i += 1;
108 bytes[i] = 'B';
109 },
110 '?' => {
111 i -= 1;
112 bytes[i] = '?';
113 },
114 'u', 'U' => {
115 // TODO unicode escape sequences
116 return error.TokenizingFailed;
117 },
118 '0'...'7' => {
119 // TODO octal escape sequences
120 return error.TokenizingFailed;
121 },
122 else => {
123 // unknown escape sequence
124 return error.TokenizingFailed;
125 },
126 }
127 i += 1;
128 escape = false;
129 } else {
130 if (c == '\\') {
131 escape = true;
132 }
133 bytes[i] = c;
134 i += 1;
135 }
136 }
137 return CToken{
138 .id = tok.id,
139 .bytes = bytes[0..i],
140 };
141}
142
143fn next(chars: [*:0]const u8, i: *usize) !CToken {
65 var state: enum {144 var state: enum {
66 Start,145 Start,
67 GotLt,146 GotLt,
...@@ -462,7 +541,7 @@ fn next(chars: [*]const u8, i: *usize) !CToken {...@@ -462,7 +541,7 @@ fn next(chars: [*]const u8, i: *usize) !CToken {
462 .String => { // TODO char escapes541 .String => { // TODO char escapes
463 switch (c) {542 switch (c) {
464 '\"' => {543 '\"' => {
465 result.bytes = chars[begin_index + 1 .. i.* - 1];544 result.bytes = chars[begin_index..i.*];
466 return result;545 return result;
467 },546 },
468 else => {},547 else => {},
...@@ -471,7 +550,7 @@ fn next(chars: [*]const u8, i: *usize) !CToken {...@@ -471,7 +550,7 @@ fn next(chars: [*]const u8, i: *usize) !CToken {
471 .CharLit => {550 .CharLit => {
472 switch (c) {551 switch (c) {
473 '\'' => {552 '\'' => {
474 result.bytes = chars[begin_index + 1 .. i.* - 1];553 result.bytes = chars[begin_index..i.*];
475 return result;554 return result;
476 },555 },
477 else => {},556 else => {},
src-self-hosted/clang.zig+1-1
...@@ -734,7 +734,7 @@ pub extern fn ZigClangSourceManager_getSpellingLoc(self: ?*const struct_ZigClang...@@ -734,7 +734,7 @@ pub extern fn ZigClangSourceManager_getSpellingLoc(self: ?*const struct_ZigClang
734pub extern fn ZigClangSourceManager_getFilename(self: *const struct_ZigClangSourceManager, SpellingLoc: struct_ZigClangSourceLocation) ?[*:0]const u8;734pub extern fn ZigClangSourceManager_getFilename(self: *const struct_ZigClangSourceManager, SpellingLoc: struct_ZigClangSourceLocation) ?[*:0]const u8;
735pub extern fn ZigClangSourceManager_getSpellingLineNumber(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) c_uint;735pub extern fn ZigClangSourceManager_getSpellingLineNumber(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) c_uint;
736pub extern fn ZigClangSourceManager_getSpellingColumnNumber(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) c_uint;736pub extern fn ZigClangSourceManager_getSpellingColumnNumber(self: ?*const struct_ZigClangSourceManager, Loc: struct_ZigClangSourceLocation) c_uint;
737pub extern fn ZigClangSourceManager_getCharacterData(self: ?*const struct_ZigClangSourceManager, SL: struct_ZigClangSourceLocation) [*c]const u8;737pub extern fn ZigClangSourceManager_getCharacterData(self: ?*const struct_ZigClangSourceManager, SL: struct_ZigClangSourceLocation) [*:0]const u8;
738pub extern fn ZigClangASTContext_getPointerType(self: ?*const struct_ZigClangASTContext, T: struct_ZigClangQualType) struct_ZigClangQualType;738pub extern fn ZigClangASTContext_getPointerType(self: ?*const struct_ZigClangASTContext, T: struct_ZigClangQualType) struct_ZigClangQualType;
739pub extern fn ZigClangASTUnit_getASTContext(self: ?*struct_ZigClangASTUnit) ?*struct_ZigClangASTContext;739pub extern fn ZigClangASTUnit_getASTContext(self: ?*struct_ZigClangASTUnit) ?*struct_ZigClangASTContext;
740pub extern fn ZigClangASTUnit_getSourceManager(self: *struct_ZigClangASTUnit) *struct_ZigClangSourceManager;740pub extern fn ZigClangASTUnit_getSourceManager(self: *struct_ZigClangASTUnit) *struct_ZigClangSourceManager;
src-self-hosted/translate_c.zig+6-14
...@@ -2629,9 +2629,9 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {...@@ -2629,9 +2629,9 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
2629 } else false;2629 } else false;
26302630
2631 (if (macro_fn)2631 (if (macro_fn)
2632 transMacroFnDefine(c, &tok_it, name, begin_c, begin_loc)2632 transMacroFnDefine(c, &tok_it, name, begin_loc)
2633 else2633 else
2634 transMacroDefine(c, &tok_it, name, begin_c, begin_loc)) catch |err| switch (err) {2634 transMacroDefine(c, &tok_it, name, begin_loc)) catch |err| switch (err) {
2635 error.UnsupportedTranslation,2635 error.UnsupportedTranslation,
2636 error.ParseError,2636 error.ParseError,
2637 => try failDecl(c, begin_loc, name, "unable to translate macro", .{}),2637 => try failDecl(c, begin_loc, name, "unable to translate macro", .{}),
...@@ -2643,7 +2643,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {...@@ -2643,7 +2643,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void {
2643 }2643 }
2644}2644}
26452645
2646fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, char_ptr: [*]const u8, source_loc: ZigClangSourceLocation) ParseError!void {2646fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
2647 const rp = makeRestorePoint(c);2647 const rp = makeRestorePoint(c);
26482648
2649 const visib_tok = try appendToken(c, .Keyword_pub, "pub");2649 const visib_tok = try appendToken(c, .Keyword_pub, "pub");
...@@ -2674,7 +2674,7 @@ fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8,...@@ -2674,7 +2674,7 @@ fn transMacroDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8,
2674 _ = try c.macro_table.put(name, &node.base);2674 _ = try c.macro_table.put(name, &node.base);
2675}2675}
26762676
2677fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, char_ptr: [*]const u8, source_loc: ZigClangSourceLocation) ParseError!void {2677fn transMacroFnDefine(c: *Context, it: *ctok.TokenList.Iterator, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void {
2678 const rp = makeRestorePoint(c);2678 const rp = makeRestorePoint(c);
2679 const pub_tok = try appendToken(c, .Keyword_pub, "pub");2679 const pub_tok = try appendToken(c, .Keyword_pub, "pub");
2680 const inline_tok = try appendToken(c, .Keyword_inline, "inline");2680 const inline_tok = try appendToken(c, .Keyword_inline, "inline");
...@@ -2829,11 +2829,7 @@ fn parseCPrimaryExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc:...@@ -2829,11 +2829,7 @@ fn parseCPrimaryExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc:
2829 const tok = it.next().?;2829 const tok = it.next().?;
2830 switch (tok.id) {2830 switch (tok.id) {
2831 .CharLit => {2831 .CharLit => {
2832 const buf = try rp.c.a().alloc(u8, tok.bytes.len + "''".len);2832 const token = try appendToken(rp.c, .CharLiteral, tok.bytes);
2833 buf[0] = '\'';
2834 writeEscapedString(buf[1..], tok.bytes);
2835 buf[buf.len - 1] = '\'';
2836 const token = try appendToken(rp.c, .CharLiteral, buf);
2837 const node = try rp.c.a().create(ast.Node.CharLiteral);2833 const node = try rp.c.a().create(ast.Node.CharLiteral);
2838 node.* = ast.Node.CharLiteral{2834 node.* = ast.Node.CharLiteral{
2839 .token = token,2835 .token = token,
...@@ -2841,11 +2837,7 @@ fn parseCPrimaryExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc:...@@ -2841,11 +2837,7 @@ fn parseCPrimaryExpr(rp: RestorePoint, it: *ctok.TokenList.Iterator, source_loc:
2841 return &node.base;2837 return &node.base;
2842 },2838 },
2843 .StrLit => {2839 .StrLit => {
2844 const buf = try rp.c.a().alloc(u8, tok.bytes.len + "\"\"".len);2840 const token = try appendToken(rp.c, .StringLiteral, tok.bytes);
2845 buf[0] = '"';
2846 writeEscapedString(buf[1..], tok.bytes);
2847 buf[buf.len - 1] = '"';
2848 const token = try appendToken(rp.c, .StringLiteral, buf);
2849 const node = try rp.c.a().create(ast.Node.StringLiteral);2841 const node = try rp.c.a().create(ast.Node.StringLiteral);
2850 node.* = ast.Node.StringLiteral{2842 node.* = ast.Node.StringLiteral{
2851 .token = token,2843 .token = token,
test/translate_c.zig+9
...@@ -411,6 +411,15 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -411,6 +411,15 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
411 \\}411 \\}
412 });412 });
413413
414 cases.add_2("macro escape sequences",
415 \\#define FOO "aoeu\xab derp"
416 \\#define FOO2 "aoeu\a derp"
417 , &[_][]const u8{
418 \\pub const FOO = "aoeu\xab derp";
419 ,
420 \\pub const FOO2 = "aoeu\x07 derp";
421 });
422
414 /////////////// Cases for only stage1 which are TODO items for stage2 ////////////////423 /////////////// Cases for only stage1 which are TODO items for stage2 ////////////////
415424
416 cases.add_both("typedef of function in struct field",425 cases.add_both("typedef of function in struct field",