authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2017-10-27 03:00:23+13:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-10-26 10:00:23-04:00
log66636381957f214f1acc22dcea01cb4cd1032649
treee74fdbb0df53d8c9d30fc04b3af4fc735732ecab
parentf4ca3482f1954eb0bdd0c378206d3a7b8e178f55

Improve invalid character error messages (#566)

See #544

2 files changed, 70 insertions(+), 9 deletions(-)

src/tokenizer.cpp+47-9
...@@ -416,6 +416,44 @@ static void handle_string_escape(Tokenize *t, uint8_t c) {...@@ -416,6 +416,44 @@ static void handle_string_escape(Tokenize *t, uint8_t c) {
416 }416 }
417}417}
418418
419static const char* get_escape_shorthand(uint8_t c) {
420 switch (c) {
421 case '\0':
422 return "\\0";
423 case '\a':
424 return "\\a";
425 case '\b':
426 return "\\b";
427 case '\t':
428 return "\\t";
429 case '\n':
430 return "\\n";
431 case '\v':
432 return "\\v";
433 case '\f':
434 return "\\f";
435 case '\r':
436 return "\\r";
437 default:
438 return nullptr;
439 }
440}
441
442static void invalid_char_error(Tokenize *t, uint8_t c) {
443 if (c == '\r') {
444 tokenize_error(t, "invalid carriage return, only '\\n' line endings are supported");
445 } else if (isprint(c)) {
446 tokenize_error(t, "invalid character: '%c'", c);
447 } else {
448 const char *sh = get_escape_shorthand(c);
449 if (sh) {
450 tokenize_error(t, "invalid character: '%s'", sh);
451 } else {
452 tokenize_error(t, "invalid character: '\\x%x'", c);
453 }
454 }
455}
456
419void tokenize(Buf *buf, Tokenization *out) {457void tokenize(Buf *buf, Tokenization *out) {
420 Tokenize t = {0};458 Tokenize t = {0};
421 t.out = out;459 t.out = out;
...@@ -580,7 +618,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -580,7 +618,7 @@ void tokenize(Buf *buf, Tokenization *out) {
580 t.state = TokenizeStateSawQuestionMark;618 t.state = TokenizeStateSawQuestionMark;
581 break;619 break;
582 default:620 default:
583 tokenize_error(&t, "invalid character: '%c'", c);621 invalid_char_error(&t, c);
584 }622 }
585 break;623 break;
586 case TokenizeStateSawQuestionMark:624 case TokenizeStateSawQuestionMark:
...@@ -890,7 +928,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -890,7 +928,7 @@ void tokenize(Buf *buf, Tokenization *out) {
890 t.state = TokenizeStateLineString;928 t.state = TokenizeStateLineString;
891 break;929 break;
892 default:930 default:
893 tokenize_error(&t, "invalid character: '%c'", c);931 invalid_char_error(&t, c);
894 break;932 break;
895 }933 }
896 break;934 break;
...@@ -919,7 +957,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -919,7 +957,7 @@ void tokenize(Buf *buf, Tokenization *out) {
919 break;957 break;
920 case '\\':958 case '\\':
921 if (t.cur_tok->data.str_lit.is_c_str) {959 if (t.cur_tok->data.str_lit.is_c_str) {
922 tokenize_error(&t, "invalid character: '%c'", c);960 invalid_char_error(&t, c);
923 }961 }
924 t.state = TokenizeStateLineStringContinue;962 t.state = TokenizeStateLineStringContinue;
925 break;963 break;
...@@ -949,7 +987,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -949,7 +987,7 @@ void tokenize(Buf *buf, Tokenization *out) {
949 buf_append_char(&t.cur_tok->data.str_lit.str, '\n');987 buf_append_char(&t.cur_tok->data.str_lit.str, '\n');
950 break;988 break;
951 default:989 default:
952 tokenize_error(&t, "invalid character: '%c'", c);990 invalid_char_error(&t, c);
953 break;991 break;
954 }992 }
955 break;993 break;
...@@ -1073,7 +1111,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1073,7 +1111,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1073 handle_string_escape(&t, '\"');1111 handle_string_escape(&t, '\"');
1074 break;1112 break;
1075 default:1113 default:
1076 tokenize_error(&t, "invalid character: '%c'", c);1114 invalid_char_error(&t, c);
1077 }1115 }
1078 break;1116 break;
1079 case TokenizeStateCharCode:1117 case TokenizeStateCharCode:
...@@ -1147,7 +1185,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1147,7 +1185,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1147 t.state = TokenizeStateStart;1185 t.state = TokenizeStateStart;
1148 break;1186 break;
1149 default:1187 default:
1150 tokenize_error(&t, "invalid character: '%c'", c);1188 invalid_char_error(&t, c);
1151 }1189 }
1152 break;1190 break;
1153 case TokenizeStateZero:1191 case TokenizeStateZero:
...@@ -1189,7 +1227,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1189,7 +1227,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1189 uint32_t digit_value = get_digit_value(c);1227 uint32_t digit_value = get_digit_value(c);
1190 if (digit_value >= t.radix) {1228 if (digit_value >= t.radix) {
1191 if (is_symbol_char(c)) {1229 if (is_symbol_char(c)) {
1192 tokenize_error(&t, "invalid character: '%c'", c);1230 invalid_char_error(&t, c);
1193 }1231 }
1194 // not my char1232 // not my char
1195 t.pos -= 1;1233 t.pos -= 1;
...@@ -1233,7 +1271,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1233,7 +1271,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1233 uint32_t digit_value = get_digit_value(c);1271 uint32_t digit_value = get_digit_value(c);
1234 if (digit_value >= t.radix) {1272 if (digit_value >= t.radix) {
1235 if (is_symbol_char(c)) {1273 if (is_symbol_char(c)) {
1236 tokenize_error(&t, "invalid character: '%c'", c);1274 invalid_char_error(&t, c);
1237 }1275 }
1238 // not my char1276 // not my char
1239 t.pos -= 1;1277 t.pos -= 1;
...@@ -1282,7 +1320,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1282,7 +1320,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1282 uint32_t digit_value = get_digit_value(c);1320 uint32_t digit_value = get_digit_value(c);
1283 if (digit_value >= t.radix) {1321 if (digit_value >= t.radix) {
1284 if (is_symbol_char(c)) {1322 if (is_symbol_char(c)) {
1285 tokenize_error(&t, "invalid character: '%c'", c);1323 invalid_char_error(&t, c);
1286 }1324 }
1287 // not my char1325 // not my char
1288 t.pos -= 1;1326 t.pos -= 1;
test/compile_errors.zig+23
...@@ -2252,4 +2252,27 @@ pub fn addCases(cases: &tests.CompileErrorContext) {...@@ -2252,4 +2252,27 @@ pub fn addCases(cases: &tests.CompileErrorContext) {
2252 \\}2252 \\}
2253 ,2253 ,
2254 ".tmp_source.zig:9:13: error: type '&MyType' does not support field access");2254 ".tmp_source.zig:9:13: error: type '&MyType' does not support field access");
2255
2256 cases.add("carriage return special case",
2257 "fn test() -> bool {\r\n" ++
2258 " true\r\n" ++
2259 "}\r\n"
2260 ,
2261 ".tmp_source.zig:1:20: error: invalid carriage return, only '\\n' line endings are supported");
2262
2263 cases.add("non-printable invalid character",
2264 "\xff\xfe" ++
2265 \\fn test() -> bool {\r
2266 \\ true\r
2267 \\}
2268 ,
2269 ".tmp_source.zig:1:1: error: invalid character: '\\xff'");
2270
2271 cases.add("non-printable invalid character with escape alternative",
2272 "fn test() -> bool {\n" ++
2273 "\ttrue\n" ++
2274 "}\n"
2275 ,
2276 ".tmp_source.zig:2:1: error: invalid character: '\\t'");
2277
2255}2278}