authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-20 13:50:34+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2019-12-20 13:50:34+02:00
loge0046b737eddffe0d6881d202608769fe720ff94
treea301c9ab53efc82d95868055d4369e1d510b3f01
parentdaeb93921049758f69474f8ad5b0d4b7a7d26a18
signature Commit is signed but in an unrecognized format.

translate-c-2 improve macro escape sequences

2 files changed, 236 insertions(+), 79 deletions(-)

src-self-hosted/c_tokenizer.zig+203-59
...@@ -74,69 +74,191 @@ fn zigifyEscapeSequences(allocator: *std.mem.Allocator, tok: CToken) !CToken {...@@ -74,69 +74,191 @@ fn zigifyEscapeSequences(allocator: *std.mem.Allocator, tok: CToken) !CToken {
74 }74 }
75 } else return tok;75 } else return tok;
76 var bytes = try allocator.alloc(u8, tok.bytes.len * 2);76 var bytes = try allocator.alloc(u8, tok.bytes.len * 2);
77 var escape = false;77 var state: enum {
78 Start,
79 Escape,
80 Hex,
81 Octal,
82 HexZero,
83 OctalZero,
84 } = .Start;
78 var i: usize = 0;85 var i: usize = 0;
86 var count: u8 = 0;
87 var num: u8 = 0;
79 for (tok.bytes) |c| {88 for (tok.bytes) |c| {
80 if (escape) {89 switch (state) {
81 switch (c) {90 .Escape => {
82 'n', 'r', 't', '\\', '\'', '\"', 'x' => {91 switch (c) {
83 bytes[i] = c;92 'n', 'r', 't', '\\', '\'', '\"' => {
84 },93 bytes[i] = c;
85 'a' => {94 },
86 bytes[i] = 'x';95 '0' => {
87 i += 1;96 state = .OctalZero;
88 bytes[i] = '0';97 bytes[i] = 'x';
89 i += 1;98 },
90 bytes[i] = '7';99 '1'...'7' => {
91 },100 count += 1;
92 'b' => {101 num *= 8;
93 bytes[i] = 'x';102 num += c - '0';
94 i += 1;103 state = .Octal;
95 bytes[i] = '0';104 bytes[i] = 'x';
96 i += 1;105 },
97 bytes[i] = '8';106 'x' => {
98 },107 state = .HexZero;
99 'f' => {108 bytes[i] = c;
100 bytes[i] = 'x';109 },
101 i += 1;110 'a' => {
102 bytes[i] = '0';111 bytes[i] = 'x';
103 i += 1;112 i += 1;
104 bytes[i] = 'C';113 bytes[i] = '0';
105 },114 i += 1;
106 'v' => {115 bytes[i] = '7';
107 bytes[i] = 'x';116 },
108 i += 1;117 'b' => {
109 bytes[i] = '0';118 bytes[i] = 'x';
110 i += 1;119 i += 1;
111 bytes[i] = 'B';120 bytes[i] = '0';
112 },121 i += 1;
113 '?' => {122 bytes[i] = '8';
114 i -= 1;123 },
115 bytes[i] = '?';124 'f' => {
116 },125 bytes[i] = 'x';
117 'u', 'U' => {126 i += 1;
118 // TODO unicode escape sequences127 bytes[i] = '0';
119 return error.TokenizingFailed;128 i += 1;
120 },129 bytes[i] = 'C';
121 '0'...'7' => {130 },
122 // TODO octal escape sequences131 'v' => {
123 return error.TokenizingFailed;132 bytes[i] = 'x';
124 },133 i += 1;
125 else => {134 bytes[i] = '0';
126 // unknown escape sequence135 i += 1;
127 return error.TokenizingFailed;136 bytes[i] = 'B';
128 },137 },
129 }138 '?' => {
130 i += 1;139 i -= 1;
131 escape = false;140 bytes[i] = '?';
132 } else {141 },
133 if (c == '\\') {142 'u', 'U' => {
134 escape = true;143 // TODO unicode escape sequences
135 }144 return error.TokenizingFailed;
136 bytes[i] = c;145 },
137 i += 1;146 else => {
147 // unknown escape sequence
148 return error.TokenizingFailed;
149 },
150 }
151 i += 1;
152 if (state == .Escape)
153 state = .Start;
154 },
155 .Start => {
156 if (c == '\\') {
157 state = .Escape;
158 }
159 bytes[i] = c;
160 i += 1;
161 },
162 .HexZero => {
163 switch (c) {
164 '0' => { continue; },
165 '1'...'9' => {
166 count += 1;
167 num *= 16;
168 num += c - '0';
169 },
170 'a'...'f' => {
171 count += 1;
172 num *= 16;
173 num += c - 'a' + 10;
174 },
175 'A'...'F' => {
176 count += 1;
177 num *= 16;
178 num += c - 'A' + 10;
179 },
180 else => {},
181 }
182 state = .Hex;
183 },
184 .Hex => {
185 switch (c) {
186 '0'...'9' => {
187 count += 1;
188 num *= 16;
189 num += c - '0';
190 if (count < 2)
191 continue;
192 },
193 'a'...'f' => {
194 count += 1;
195 num *= 16;
196 num += c - 'a' + 10;
197 if (count < 2)
198 continue;
199 },
200 'A'...'F' => {
201 count += 1;
202 num *= 16;
203 num += c - 'A' + 10;
204 if (count < 2)
205 continue;
206 },
207 else => {},
208 }
209 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{.fill = '0', .width = 2});
210 switch (c) {
211 '\\' => state = .Escape,
212 '0'...'9', 'a'...'f','A'...'F' => state = .Start,
213 else => {
214 state = .Start;
215 bytes[i] = c;
216 i += 1;
217 },
218 }
219 count = 0;
220 num = 0;
221 },
222 .OctalZero => {
223 switch (c) {
224 '0' => { continue; },
225 '1'...'7' => {
226 count += 1;
227 num *= 8;
228 num += c - '0';
229 },
230 else => {},
231 }
232 state = .Octal;
233 },
234 .Octal => {
235 switch (c) {
236 '0'...'7' => {
237 count += 1;
238 num *= 8;
239 num += c - '0';
240 if (count < 3)
241 continue;
242 },
243 else => {},
244 }
245 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{.fill = '0', .width = 2});
246 switch (c) {
247 '\\' => state = .Escape,
248 '0'...'7' => state = .Start,
249 else => {
250 state = .Start;
251 bytes[i] = c;
252 i += 1;
253 },
254 }
255 count = 0;
256 num = 0;
257 },
138 }258 }
139 }259 }
260 if (state == .Hex or state == .Octal)
261 i += std.fmt.formatIntBuf(bytes[i..], num, 16, false, std.fmt.FormatOptions{.fill = '0', .width = 2});
140 return CToken{262 return CToken{
141 .id = tok.id,263 .id = tok.id,
142 .bytes = bytes[0..i],264 .bytes = bytes[0..i],
...@@ -666,3 +788,25 @@ test "tokenize macro" {...@@ -666,3 +788,25 @@ test "tokenize macro" {
666 expect(it.next() == null);788 expect(it.next() == null);
667 tl.shrink(0);789 tl.shrink(0);
668}790}
791
792test "escape sequences" {
793 var buf: [1024]u8 = undefined;
794 var alloc = std.heap.FixedBufferAllocator.init(buf[0..]);
795 const a = &alloc.allocator;
796 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
797 .id = .StrLit,
798 .bytes = "\\x0077",
799 })).bytes, "\\x77"));
800 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
801 .id = .StrLit,
802 .bytes = "\\00245",
803 })).bytes, "\\xa5"));
804 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
805 .id = .StrLit,
806 .bytes = "\\x0077abc",
807 })).bytes, "\\x77abc"));
808 expect(std.mem.eql(u8, (try zigifyEscapeSequences(a, .{
809 .id = .StrLit,
810 .bytes = "\\045abc",
811 })).bytes, "\\x25abc"));
812}
test/translate_c.zig+33-20
...@@ -1089,13 +1089,16 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -1089,13 +1089,16 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
1089 \\}1089 \\}
1090 });1090 });
10911091
1092 cases.add_2("macro escape sequences",1092 cases.add_2("macro defines string literal with hex",
1093 \\#define FOO "aoeu\xab derp"1093 \\#define FOO "aoeu\xab derp"
1094 \\#define FOO2 "aoeu\a derp"1094 \\#define FOO2 "aoeu\x0007a derp"
1095 \\#define FOO_CHAR '\xfF'
1095 , &[_][]const u8{1096 , &[_][]const u8{
1096 \\pub const FOO = "aoeu\xab derp";1097 \\pub const FOO = "aoeu\xab derp";
1097 ,1098 ,
1098 \\pub const FOO2 = "aoeu\x07 derp";1099 \\pub const FOO2 = "aoeu\x7a derp";
1100 ,
1101 \\pub const FOO_CHAR = '\xff';
1099 });1102 });
11001103
1101 cases.add_2("variable aliasing",1104 cases.add_2("variable aliasing",
...@@ -2157,30 +2160,16 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -2157,30 +2160,16 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
2157 \\}2160 \\}
2158 });2161 });
21592162
2160 /////////////// Cases for only stage1 which are TODO items for stage2 ////////////////2163 cases.add_2("macro defines string literal with octal",
2161
2162 cases.add("macro defines string literal with hex",
2163 \\#define FOO "aoeu\xab derp"
2164 \\#define FOO2 "aoeu\x0007a derp"
2165 \\#define FOO_CHAR '\xfF'
2166 , &[_][]const u8{
2167 \\pub const FOO = "aoeu\xab derp";
2168 ,
2169 \\pub const FOO2 = "aoeuz derp";
2170 ,
2171 \\pub const FOO_CHAR = 255;
2172 });
2173
2174 cases.add("macro defines string literal with octal",
2175 \\#define FOO "aoeu\023 derp"2164 \\#define FOO "aoeu\023 derp"
2176 \\#define FOO2 "aoeu\0234 derp"2165 \\#define FOO2 "aoeu\0234 derp"
2177 \\#define FOO_CHAR '\077'2166 \\#define FOO_CHAR '\077'
2178 , &[_][]const u8{2167 , &[_][]const u8{
2179 \\pub const FOO = "aoeu\x13 derp";2168 \\pub const FOO = "aoeu\x13 derp";
2180 ,2169 ,
2181 \\pub const FOO2 = "aoeu\x134 derp";2170 \\pub const FOO2 = "aoeu\x9c derp";
2182 ,2171 ,
2183 \\pub const FOO_CHAR = 63;2172 \\pub const FOO_CHAR = '\x3f';
2184 });2173 });
21852174
2186 /////////////// Cases for only stage1 because stage2 behavior is better ////////////////2175 /////////////// Cases for only stage1 because stage2 behavior is better ////////////////
...@@ -3111,4 +3100,28 @@ pub fn addCases(cases: *tests.TranslateCContext) void {...@@ -3111,4 +3100,28 @@ pub fn addCases(cases: *tests.TranslateCContext) void {
3111 \\ _ = baz.?();3100 \\ _ = baz.?();
3112 \\}3101 \\}
3113 });3102 });
3103
3104 cases.add("macro defines string literal with hex",
3105 \\#define FOO "aoeu\xab derp"
3106 \\#define FOO2 "aoeu\x0007a derp"
3107 \\#define FOO_CHAR '\xfF'
3108 , &[_][]const u8{
3109 \\pub const FOO = "aoeu\xab derp";
3110 ,
3111 \\pub const FOO2 = "aoeuz derp";
3112 ,
3113 \\pub const FOO_CHAR = 255;
3114 });
3115
3116 cases.add("macro defines string literal with octal",
3117 \\#define FOO "aoeu\023 derp"
3118 \\#define FOO2 "aoeu\0234 derp"
3119 \\#define FOO_CHAR '\077'
3120 , &[_][]const u8{
3121 \\pub const FOO = "aoeu\x13 derp";
3122 ,
3123 \\pub const FOO2 = "aoeu\x134 derp";
3124 ,
3125 \\pub const FOO_CHAR = 63;
3126 });
3114}3127}