authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-20 13:50:34+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-20 13:50:34+02:00
loge0046b737eddffe0d6881d202608769fe720ff94
treea301c9ab53efc82d95868055d4369e1d510b3f01
parentdaeb93921049758f69474f8ad5b0d4b7a7d26a18
signature Commit is signed but in an unrecognized format.

translate-c-2 improve macro escape sequences

2 files changed, 236 insertions(+), 79 deletions(-)

src-self-hosted/c_tokenizer.zig+203-59
......@@ -74,69 +74,191 @@ fn zigifyEscapeSequences(allocator: *std.mem.Allocator, tok: CToken) !CToken {
7474 }
7575 } else return tok;
7676 var bytes = try allocator.alloc(u8, tok.bytes.len * 2);
77 var escape = false;
77 var state: enum {
78 Start,
79 Escape,
80 Hex,
81 Octal,
82 HexZero,
83 OctalZero,
84 } = .Start;
7885 var i: usize = 0;
86 var count: u8 = 0;
87 var num: u8 = 0;
7988 for (tok.bytes) |c| {
80 if (escape) {
81 switch (c) {
82 'n', 'r', 't', '\\', '\'', '\"', 'x' => {
83 bytes[i] = c;
84 },
85 'a' => {
86 bytes[i] = 'x';
87 i += 1;
88 bytes[i] = '0';
89 i += 1;
90 bytes[i] = '7';
91 },
92 'b' => {
93 bytes[i] = 'x';
94 i += 1;
95 bytes[i] = '0';
96 i += 1;
97 bytes[i] = '8';
98 },
99 'f' => {
100 bytes[i] = 'x';
101 i += 1;
102 bytes[i] = '0';
103 i += 1;
104 bytes[i] = 'C';
105 },
106 'v' => {
107 bytes[i] = 'x';
108 i += 1;
109 bytes[i] = '0';
110 i += 1;
111 bytes[i] = 'B';
112 },
113 '?' => {
114 i -= 1;
115 bytes[i] = '?';
116 },
117 'u', 'U' => {
118 // TODO unicode escape sequences
119 return error.TokenizingFailed;
120 },
121 '0'...'7' => {
122 // TODO octal escape sequences
123 return error.TokenizingFailed;
124 },
125 else => {
126 // unknown escape sequence
127 return error.TokenizingFailed;
128 },
129 }
130 i += 1;
131 escape = false;
132 } else {
133 if (c == '\\') {
134 escape = true;
135 }
136 bytes[i] = c;
137 i += 1;
89 switch (state) {
90 .Escape => {
91 switch (c) {
92 'n', 'r', 't', '\\', '\'', '\"' => {
93 bytes[i] = c;
94 },
95 '0' => {
96 state = .OctalZero;
97 bytes[i] = 'x';
98 },
99 '1'...'7' => {
100 count += 1;
101 num *= 8;
102 num += c - '0';
103 state = .Octal;
104 bytes[i] = 'x';
105 },
106 'x' => {
107 state = .HexZero;
108 bytes[i] = c;
109 },
110 'a' => {
111 bytes[i] = 'x';
112 i += 1;
113 bytes[i] = '0';
114 i += 1;
115 bytes[i] = '7';
116 },
117 'b' => {
118 bytes[i] = 'x';
119 i += 1;
120 bytes[i] = '0';
121 i += 1;
122 bytes[i] = '8';
123 },
124 'f' => {
125 bytes[i] = 'x';
126 i += 1;
127 bytes[i] = '0';
128 i += 1;
129 bytes[i] = 'C';
130 },
131 'v' => {
132 bytes[i] = 'x';
133 i += 1;
134 bytes[i] = '0';
135 i += 1;
136 bytes[i] = 'B';
137 },
138 '?' => {
139 i -= 1;
140 bytes[i] = '?';
141 },
142 'u', 'U' => {
143 // TODO unicode escape sequences
144 return error.TokenizingFailed;
145 },
146 else => {
147 // unknown escape sequence
148 return error.TokenizingFailed;
149 },
150 }
151 i += 1;
152 if (state == .Escape)
153 state = .Start;
154 },
155 .Start => {
156 if (c == '\\') {
157 state = .Escape;
158 }
159 bytes[i] = c;
160 i += 1;
161 },
162 .HexZero => {
163 switch (c) {
164 '0' => { continue; },
165 '1'...'9' => {
166 count += 1;
167 num *= 16;
168 num += c - '0';
169 },
170 'a'...'f' => {
171 count += 1;
172 num *= 16;
173 num += c - 'a' + 10;
174 },
175 'A'...'F' => {
176 count += 1;
177 num *= 16;
178 num += c - 'A' + 10;
179 },
180 else => {},
181 }
182 state = .Hex;
183 },
184 .Hex => {
185 switch (c) {
186 '0'...'9' => {
187 count += 1;
188 num *= 16;
189 num += c - '0';
190 if (count < 2)
191 continue;
192 },
193 'a'...'f' => {
194 count += 1;
195 num *= 16;
196 num += c - 'a' + 10;
197 if (count < 2)
198 continue;
199 },
200 'A'...'F' => {
201 count += 1;
202 num *= 16;
203 num += c - 'A' + 10;
204 if (count < 2)
205 continue;
206 },
207 else => {},
208 }
209 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{.fill = '0', .width = 2});
210 switch (c) {
211 '\\' => state = .Escape,
212 '0'...'9', 'a'...'f','A'...'F' => state = .Start,
213 else => {
214 state = .Start;
215 bytes[i] = c;
216 i += 1;
217 },
218 }
219 count = 0;
220 num = 0;
221 },
222 .OctalZero => {
223 switch (c) {
224 '0' => { continue; },
225 '1'...'7' => {
226 count += 1;
227 num *= 8;
228 num += c - '0';
229 },
230 else => {},
231 }
232 state = .Octal;
233 },
234 .Octal => {
235 switch (c) {
236 '0'...'7' => {
237 count += 1;
238 num *= 8;
239 num += c - '0';
240 if (count < 3)
241 continue;
242 },
243 else => {},
244 }
245 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{.fill = '0', .width = 2});
246 switch (c) {
247 '\\' => state = .Escape,
248 '0'...'7' => state = .Start,
249 else => {
250 state = .Start;
251 bytes[i] = c;
252 i += 1;
253 },
254 }
255 count = 0;
256 num = 0;
257 },
138258 }
139259 }
260 if (state == .Hex or state == .Octal)
261 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{.fill = '0', .width = 2});
140262 return CToken{
141263 .id = tok.id,
142264 .bytes = bytes[0..i],
......@@ -666,3 +788,25 @@ test "tokenize macro" {
666788 expect(it.next() == null);
667789 tl.shrink(0);
668790}
791
792test "escape sequences" {
793 var buf: [1024]u8 = undefined;
794 var alloc = std.heap.FixedBufferAllocator.init(buf[0..]);
795 const a = &alloc.allocator;
796 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
797 .id = .StrLit,
798 .bytes = "\\x0077",
799 })).bytes, "\\x77"));
800 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
801 .id = .StrLit,
802 .bytes = "\\00245",
803 })).bytes, "\\xa5"));
804 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
805 .id = .StrLit,
806 .bytes = "\\x0077abc",
807 })).bytes, "\\x77abc"));
808 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
809 .id = .StrLit,
810 .bytes = "\\045abc",
811 })).bytes, "\\x25abc"));
812}
test/translate_c.zig+33-20
......@@ -1089,13 +1089,16 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
10891089 \\}
10901090 });
10911091
1092 cases.add_2("macro escape sequences",
1092 cases.add_2("macro defines string literal with hex",
10931093 \\#define FOO "aoeu\xab derp"
1094 \\#define FOO2 "aoeu\a derp"
1094 \\#define FOO2 "aoeu\x0007a derp"
1095 \\#define FOO_CHAR '\xfF'
10951096 , &[_][]const u8{
10961097 \\pub const FOO = "aoeu\xab derp";
10971098 ,
1098 \\pub const FOO2 = "aoeu\x07 derp";
1099 \\pub const FOO2 = "aoeu\x7a derp";
1100 ,
1101 \\pub const FOO_CHAR = '\xff';
10991102 });
11001103
11011104 cases.add_2("variable aliasing",
......@@ -2157,30 +2160,16 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
21572160 \\}
21582161 });
21592162
2160 /////////////// Cases for only stage1 which are TODO items for stage2 ////////////////
2161
2162 cases.add("macro defines string literal with hex",
2163 \\#define FOO "aoeu\xab derp"
2164 \\#define FOO2 "aoeu\x0007a derp"
2165 \\#define FOO_CHAR '\xfF'
2166 , &[_][]const u8{
2167 \\pub const FOO = "aoeu\xab derp";
2168 ,
2169 \\pub const FOO2 = "aoeuz derp";
2170 ,
2171 \\pub const FOO_CHAR = 255;
2172 });
2173
2174 cases.add("macro defines string literal with octal",
2163 cases.add_2("macro defines string literal with octal",
21752164 \\#define FOO "aoeu\023 derp"
21762165 \\#define FOO2 "aoeu\0234 derp"
21772166 \\#define FOO_CHAR '\077'
21782167 , &[_][]const u8{
21792168 \\pub const FOO = "aoeu\x13 derp";
21802169 ,
2181 \\pub const FOO2 = "aoeu\x134 derp";
2170 \\pub const FOO2 = "aoeu\x9c derp";
21822171 ,
2183 \\pub const FOO_CHAR = 63;
2172 \\pub const FOO_CHAR = '\x3f';
21842173 });
21852174
21862175 /////////////// Cases for only stage1 because stage2 behavior is better ////////////////
......@@ -3111,4 +3100,28 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
31113100 \\ _ = baz.?();
31123101 \\}
31133102 });
3103
3104 cases.add("macro defines string literal with hex",
3105 \\#define FOO "aoeu\xab derp"
3106 \\#define FOO2 "aoeu\x0007a derp"
3107 \\#define FOO_CHAR '\xfF'
3108 , &[_][]const u8{
3109 \\pub const FOO = "aoeu\xab derp";
3110 ,
3111 \\pub const FOO2 = "aoeuz derp";
3112 ,
3113 \\pub const FOO_CHAR = 255;
3114 });
3115
3116 cases.add("macro defines string literal with octal",
3117 \\#define FOO "aoeu\023 derp"
3118 \\#define FOO2 "aoeu\0234 derp"
3119 \\#define FOO_CHAR '\077'
3120 , &[_][]const u8{
3121 \\pub const FOO = "aoeu\x13 derp";
3122 ,
3123 \\pub const FOO2 = "aoeu\x134 derp";
3124 ,
3125 \\pub const FOO_CHAR = 63;
3126 });
31143127}