authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-06-16 14:34:38-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-06-16 14:34:38-04:00
logc0f9012bed2e0646d15d454cffb971f4b7368edf
tree643c04eac9c5f091c9f9b57c1114f05ca9953bb0
parent865b53f2860405a718262abf9a794d2bf9529dbc

parseh: fix not recognizing integer suffixes on hex numbers

4 files changed, 189 insertions(+), 71 deletions(-)

src/c_tokenizer.cpp+96-63
......@@ -107,8 +107,11 @@ static void begin_token(CTokenize *ctok, CTokId id) {
107107 memset(&ctok->cur_tok->data.symbol, 0, sizeof(Buf));
108108 buf_resize(&ctok->cur_tok->data.symbol, 0);
109109 break;
110 case CTokIdCharLit:
111110 case CTokIdNumLitInt:
111 ctok->cur_tok->data.num_lit_int.x = 0;
112 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixNone;
113 break;
114 case CTokIdCharLit:
112115 case CTokIdNumLitFloat:
113116 case CTokIdMinus:
114117 break;
......@@ -138,9 +141,9 @@ static void add_char(CTokenize *ctok, uint8_t c) {
138141
139142static void hex_digit(CTokenize *ctok, uint8_t value) {
140143 // TODO @mul_with_overflow
141 ctok->cur_tok->data.num_lit_int *= 16;
144 ctok->cur_tok->data.num_lit_int.x *= 16;
142145 // TODO @add_with_overflow
143 ctok->cur_tok->data.num_lit_int += value;
146 ctok->cur_tok->data.num_lit_int.x += value;
144147
145148 static const uint8_t hex_digit[] = "0123456789abcdef";
146149 buf_append_char(&ctok->buf, hex_digit[value]);
......@@ -194,19 +197,15 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
194197 break;
195198 case DIGIT_NON_ZERO:
196199 ctok->state = CTokStateDecimal;
197 ctok->unsigned_suffix = false;
198 ctok->long_suffix = false;
199200 begin_token(ctok, CTokIdNumLitInt);
200 ctok->cur_tok->data.num_lit_int = *c - '0';
201 ctok->cur_tok->data.num_lit_int.x = *c - '0';
201202 buf_resize(&ctok->buf, 0);
202203 buf_append_char(&ctok->buf, *c);
203204 break;
204205 case '0':
205206 ctok->state = CTokStateGotZero;
206 ctok->unsigned_suffix = false;
207 ctok->long_suffix = false;
208207 begin_token(ctok, CTokIdNumLitInt);
209 ctok->cur_tok->data.num_lit_int = 0;
208 ctok->cur_tok->data.num_lit_int.x = 0;
210209 buf_resize(&ctok->buf, 0);
211210 buf_append_char(&ctok->buf, '0');
212211 break;
......@@ -289,21 +288,21 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
289288 buf_append_char(&ctok->buf, *c);
290289
291290 // TODO @mul_with_overflow
292 ctok->cur_tok->data.num_lit_int *= 10;
291 ctok->cur_tok->data.num_lit_int.x *= 10;
293292 // TODO @add_with_overflow
294 ctok->cur_tok->data.num_lit_int += *c - '0';
293 ctok->cur_tok->data.num_lit_int.x += *c - '0';
295294 break;
296295 case '\'':
297296 break;
298297 case 'u':
299298 case 'U':
300 ctok->unsigned_suffix = true;
301 ctok->state = CTokStateIntSuffix;
299 ctok->state = CTokStateNumLitIntSuffixU;
300 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixU;
302301 break;
303302 case 'l':
304303 case 'L':
305 ctok->long_suffix = true;
306 ctok->state = CTokStateIntSuffixLong;
304 ctok->state = CTokStateNumLitIntSuffixL;
305 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixL;
307306 break;
308307 case '.':
309308 buf_append_char(&ctok->buf, '.');
......@@ -317,50 +316,6 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
317316 continue;
318317 }
319318 break;
320 case CTokStateIntSuffix:
321 switch (*c) {
322 case 'l':
323 case 'L':
324 if (ctok->long_suffix) {
325 return mark_error(ctok);
326 }
327 ctok->long_suffix = true;
328 ctok->state = CTokStateIntSuffixLong;
329 break;
330 case 'u':
331 case 'U':
332 if (ctok->unsigned_suffix) {
333 return mark_error(ctok);
334 }
335 ctok->unsigned_suffix = true;
336 break;
337 default:
338 c -= 1;
339 end_token(ctok);
340 ctok->state = CTokStateStart;
341 continue;
342 }
343 break;
344 case CTokStateIntSuffixLong:
345 switch (*c) {
346 case 'l':
347 case 'L':
348 ctok->state = CTokStateIntSuffix;
349 break;
350 case 'u':
351 case 'U':
352 if (ctok->unsigned_suffix) {
353 return mark_error(ctok);
354 }
355 ctok->unsigned_suffix = true;
356 break;
357 default:
358 c -= 1;
359 end_token(ctok);
360 ctok->state = CTokStateStart;
361 continue;
362 }
363 break;
364319 case CTokStateGotZero:
365320 switch (*c) {
366321 case 'x':
......@@ -389,9 +344,9 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
389344 case '6':
390345 case '7':
391346 // TODO @mul_with_overflow
392 ctok->cur_tok->data.num_lit_int *= 8;
347 ctok->cur_tok->data.num_lit_int.x *= 8;
393348 // TODO @add_with_overflow
394 ctok->cur_tok->data.num_lit_int += *c - '0';
349 ctok->cur_tok->data.num_lit_int.x += *c - '0';
395350 break;
396351 case '8':
397352 case '9':
......@@ -466,6 +421,82 @@ void tokenize_c_macro(CTokenize *ctok, const uint8_t *c) {
466421 ctok->cur_tok->id = CTokIdNumLitFloat;
467422 ctok->state = CTokStateExpSign;
468423 break;
424 case 'u':
425 case 'U':
426 // marks the number literal as unsigned
427 ctok->state = CTokStateNumLitIntSuffixU;
428 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixU;
429 break;
430 case 'l':
431 case 'L':
432 // marks the number literal as long
433 ctok->state = CTokStateNumLitIntSuffixL;
434 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixL;
435 break;
436 default:
437 c -= 1;
438 end_token(ctok);
439 ctok->state = CTokStateStart;
440 continue;
441 }
442 break;
443 case CTokStateNumLitIntSuffixU:
444 switch (*c) {
445 case 'l':
446 case 'L':
447 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLU;
448 ctok->state = CTokStateNumLitIntSuffixUL;
449 break;
450 default:
451 c -= 1;
452 end_token(ctok);
453 ctok->state = CTokStateStart;
454 continue;
455 }
456 break;
457 case CTokStateNumLitIntSuffixL:
458 switch (*c) {
459 case 'l':
460 case 'L':
461 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLL;
462 ctok->state = CTokStateNumLitIntSuffixLL;
463 break;
464 case 'u':
465 case 'U':
466 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLU;
467 end_token(ctok);
468 ctok->state = CTokStateStart;
469 break;
470 default:
471 c -= 1;
472 end_token(ctok);
473 ctok->state = CTokStateStart;
474 continue;
475 }
476 break;
477 case CTokStateNumLitIntSuffixLL:
478 switch (*c) {
479 case 'u':
480 case 'U':
481 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLLU;
482 end_token(ctok);
483 ctok->state = CTokStateStart;
484 break;
485 default:
486 c -= 1;
487 end_token(ctok);
488 ctok->state = CTokStateStart;
489 continue;
490 }
491 break;
492 case CTokStateNumLitIntSuffixUL:
493 switch (*c) {
494 case 'l':
495 case 'L':
496 ctok->cur_tok->data.num_lit_int.suffix = CNumLitSuffixLLU;
497 end_token(ctok);
498 ctok->state = CTokStateStart;
499 break;
469500 default:
470501 c -= 1;
471502 end_token(ctok);
......@@ -681,8 +712,10 @@ found_end_of_macro:
681712 case CTokStateHex:
682713 case CTokStateOctal:
683714 case CTokStateGotZero:
684 case CTokStateIntSuffix:
685 case CTokStateIntSuffixLong:
715 case CTokStateNumLitIntSuffixU:
716 case CTokStateNumLitIntSuffixL:
717 case CTokStateNumLitIntSuffixUL:
718 case CTokStateNumLitIntSuffixLL:
686719 end_token(ctok);
687720 break;
688721 case CTokStateFloat:
src/c_tokenizer.hpp+19-5
......@@ -20,12 +20,26 @@ enum CTokId {
2020 CTokIdMinus,
2121};
2222
23enum CNumLitSuffix {
24 CNumLitSuffixNone,
25 CNumLitSuffixL,
26 CNumLitSuffixU,
27 CNumLitSuffixLU,
28 CNumLitSuffixLL,
29 CNumLitSuffixLLU,
30};
31
32struct CNumLitInt {
33 uint64_t x;
34 CNumLitSuffix suffix;
35};
36
2337struct CTok {
2438 enum CTokId id;
2539 union {
2640 uint8_t char_lit;
2741 Buf str_lit;
28 uint64_t num_lit_int;
42 CNumLitInt num_lit_int;
2943 double num_lit_float;
3044 Buf symbol;
3145 } data;
......@@ -47,13 +61,15 @@ enum CTokState {
4761 CTokStateOctal,
4862 CTokStateGotZero,
4963 CTokStateHex,
50 CTokStateIntSuffix,
51 CTokStateIntSuffixLong,
5264 CTokStateFloat,
5365 CTokStateExpSign,
5466 CTokStateFloatExp,
5567 CTokStateFloatExpFirst,
5668 CTokStateStrOctal,
69 CTokStateNumLitIntSuffixU,
70 CTokStateNumLitIntSuffixL,
71 CTokStateNumLitIntSuffixLL,
72 CTokStateNumLitIntSuffixUL,
5773};
5874
5975struct CTokenize {
......@@ -62,8 +78,6 @@ struct CTokenize {
6278 bool error;
6379 CTok *cur_tok;
6480 Buf buf;
65 bool unsigned_suffix;
66 bool long_suffix;
6781 uint8_t cur_char;
6882 int octal_index;
6983};
src/parseh.cpp+32-3
......@@ -162,12 +162,16 @@ static Tld *create_global_str_lit_var(Context *c, Buf *name, Buf *value) {
162162 return &tld_var->base;
163163}
164164
165static Tld *create_global_num_lit_unsigned_negative(Context *c, Buf *name, uint64_t x, bool negative) {
166 ConstExprValue *var_val = create_const_unsigned_negative(c->codegen->builtin_types.entry_num_lit_int, x, negative);
165static Tld *create_global_num_lit_unsigned_negative_type(Context *c, Buf *name, uint64_t x, bool negative, TypeTableEntry *type_entry) {
166 ConstExprValue *var_val = create_const_unsigned_negative(type_entry, x, negative);
167167 TldVar *tld_var = create_global_var(c, name, var_val, true);
168168 return &tld_var->base;
169169}
170170
171static Tld *create_global_num_lit_unsigned_negative(Context *c, Buf *name, uint64_t x, bool negative) {
172 return create_global_num_lit_unsigned_negative_type(c, name, x, negative, c->codegen->builtin_types.entry_num_lit_int);
173}
174
171175static Tld *create_global_num_lit_float(Context *c, Buf *name, double value) {
172176 ConstExprValue *var_val = create_const_float(c->codegen->builtin_types.entry_num_lit_float, value);
173177 TldVar *tld_var = create_global_var(c, name, var_val, true);
......@@ -1149,7 +1153,32 @@ static void process_macro(Context *c, CTokenize *ctok, Buf *name, const char *ch
11491153 return;
11501154 case CTokIdNumLitInt:
11511155 if (is_last) {
1152 Tld *tld = create_global_num_lit_unsigned_negative(c, name, tok->data.num_lit_int, negate);
1156 Tld *tld;
1157 switch (tok->data.num_lit_int.suffix) {
1158 case CNumLitSuffixNone:
1159 tld = create_global_num_lit_unsigned_negative(c, name, tok->data.num_lit_int.x, negate);
1160 break;
1161 case CNumLitSuffixL:
1162 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1163 c->codegen->builtin_types.entry_c_int[CIntTypeLong]);
1164 break;
1165 case CNumLitSuffixU:
1166 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1167 c->codegen->builtin_types.entry_c_int[CIntTypeUInt]);
1168 break;
1169 case CNumLitSuffixLU:
1170 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1171 c->codegen->builtin_types.entry_c_int[CIntTypeULong]);
1172 break;
1173 case CNumLitSuffixLL:
1174 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1175 c->codegen->builtin_types.entry_c_int[CIntTypeLongLong]);
1176 break;
1177 case CNumLitSuffixLLU:
1178 tld = create_global_num_lit_unsigned_negative_type(c, name, tok->data.num_lit_int.x, negate,
1179 c->codegen->builtin_types.entry_c_int[CIntTypeULongLong]);
1180 break;
1181 }
11531182 c->macro_table.put(name, tld);
11541183 }
11551184 return;
test/parseh.zig+42
......@@ -217,6 +217,48 @@ pub fn addCases(cases: &tests.ParseHContext) {
217217 \\pub const SDL_INIT_VIDEO = 32;
218218 );
219219
220 cases.add("u integer suffix after hex literal",
221 \\#define SDL_INIT_VIDEO 0x00000020u /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
222 ,
223 \\pub const SDL_INIT_VIDEO: c_uint = 32;
224 );
225
226 cases.add("l integer suffix after hex literal",
227 \\#define SDL_INIT_VIDEO 0x00000020l /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
228 ,
229 \\pub const SDL_INIT_VIDEO: c_long = 32;
230 );
231
232 cases.add("ul integer suffix after hex literal",
233 \\#define SDL_INIT_VIDEO 0x00000020ul /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
234 ,
235 \\pub const SDL_INIT_VIDEO: c_ulong = 32;
236 );
237
238 cases.add("lu integer suffix after hex literal",
239 \\#define SDL_INIT_VIDEO 0x00000020lu /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
240 ,
241 \\pub const SDL_INIT_VIDEO: c_ulong = 32;
242 );
243
244 cases.add("ll integer suffix after hex literal",
245 \\#define SDL_INIT_VIDEO 0x00000020ll /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
246 ,
247 \\pub const SDL_INIT_VIDEO: c_longlong = 32;
248 );
249
250 cases.add("ull integer suffix after hex literal",
251 \\#define SDL_INIT_VIDEO 0x00000020ull /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
252 ,
253 \\pub const SDL_INIT_VIDEO: c_ulonglong = 32;
254 );
255
256 cases.add("llu integer suffix after hex literal",
257 \\#define SDL_INIT_VIDEO 0x00000020llu /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
258 ,
259 \\pub const SDL_INIT_VIDEO: c_ulonglong = 32;
260 );
261
220262 cases.add("zig keywords in C code",
221263 \\struct comptime {
222264 \\ int defer;