authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-22 15:24:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-22 15:24:18-07:00
log8187396f640d2a56e0d56c7d199074b4590e49eb
tree5dbbeeba42680a30c43ccd1d1be3be89925d73d3
parent35362f8137b2c5109e6bc39cb12048c016b5b580

add syntax to allow symbols to have arbitrary strings as names

6 files changed, 208 insertions(+), 95 deletions(-)

src/ast_render.cpp+83-11
......@@ -239,10 +239,80 @@ static bool is_node_void(AstNode *node) {
239239 return false;
240240}
241241
242static bool is_printable(uint8_t c) {
242static bool is_alpha_under(uint8_t c) {
243243 return (c >= 'a' && c <= 'z') ||
244 (c >= 'A' && c <= 'A') ||
245 (c >= '0' && c <= '9');
244 (c >= 'A' && c <= 'Z') || c == '_';
245}
246
247static bool is_digit(uint8_t c) {
248 return (c >= '0' && c <= '9');
249}
250
251static bool is_printable(uint8_t c) {
252 return is_alpha_under(c) || is_digit(c);
253}
254
255static void string_literal_escape(Buf *source, Buf *dest) {
256 buf_resize(dest, 0);
257 for (int i = 0; i < buf_len(source); i += 1) {
258 uint8_t c = *((uint8_t*)buf_ptr(source) + i);
259 if (is_printable(c)) {
260 buf_append_char(dest, c);
261 } else if (c == '\'') {
262 buf_append_str(dest, "\\'");
263 } else if (c == '"') {
264 buf_append_str(dest, "\\\"");
265 } else if (c == '\\') {
266 buf_append_str(dest, "\\\\");
267 } else if (c == '\a') {
268 buf_append_str(dest, "\\a");
269 } else if (c == '\b') {
270 buf_append_str(dest, "\\b");
271 } else if (c == '\f') {
272 buf_append_str(dest, "\\f");
273 } else if (c == '\n') {
274 buf_append_str(dest, "\\n");
275 } else if (c == '\r') {
276 buf_append_str(dest, "\\r");
277 } else if (c == '\t') {
278 buf_append_str(dest, "\\t");
279 } else if (c == '\v') {
280 buf_append_str(dest, "\\v");
281 } else {
282 buf_appendf(dest, "\\x%x", (int)c);
283 }
284 }
285}
286
287static bool is_valid_bare_symbol(Buf *symbol) {
288 if (buf_len(symbol) == 0) {
289 return false;
290 }
291 uint8_t first_char = *buf_ptr(symbol);
292 if (!is_alpha_under(first_char)) {
293 return false;
294 }
295 for (int i = 1; i < buf_len(symbol); i += 1) {
296 uint8_t c = *((uint8_t*)buf_ptr(symbol) + i);
297 if (!is_alpha_under(c) && !is_digit(c)) {
298 return false;
299 }
300 }
301 return true;
302}
303
304static void print_symbol(AstRender *ar, Buf *symbol) {
305 if (is_zig_keyword(symbol)) {
306 fprintf(ar->f, "@\"%s\"", buf_ptr(symbol));
307 return;
308 }
309 if (is_valid_bare_symbol(symbol)) {
310 fprintf(ar->f, "%s", buf_ptr(symbol));
311 return;
312 }
313 Buf escaped = BUF_INIT;
314 string_literal_escape(symbol, &escaped);
315 fprintf(ar->f, "@\"%s\"", buf_ptr(&escaped));
246316}
247317
248318static void render_node(AstRender *ar, AstNode *node) {
......@@ -268,20 +338,22 @@ static void render_node(AstRender *ar, AstNode *node) {
268338 break;
269339 case NodeTypeFnProto:
270340 {
271 const char *fn_name = buf_ptr(&node->data.fn_proto.name);
272341 const char *pub_str = visib_mod_string(node->data.fn_proto.top_level_decl.visib_mod);
273342 const char *extern_str = extern_string(node->data.fn_proto.is_extern);
274343 const char *inline_str = inline_string(node->data.fn_proto.is_inline);
275 fprintf(ar->f, "%s%s%sfn %s(", pub_str, inline_str, extern_str, fn_name);
344 fprintf(ar->f, "%s%s%sfn ", pub_str, inline_str, extern_str);
345 print_symbol(ar, &node->data.fn_proto.name);
346 fprintf(ar->f, "(");
276347 int arg_count = node->data.fn_proto.params.length;
277348 bool is_var_args = node->data.fn_proto.is_var_args;
278349 for (int arg_i = 0; arg_i < arg_count; arg_i += 1) {
279350 AstNode *param_decl = node->data.fn_proto.params.at(arg_i);
280351 assert(param_decl->type == NodeTypeParamDecl);
281 const char *arg_name = buf_ptr(&param_decl->data.param_decl.name);
282352 if (buf_len(&param_decl->data.param_decl.name) > 0) {
283353 const char *noalias_str = param_decl->data.param_decl.is_noalias ? "noalias " : "";
284 fprintf(ar->f, "%s%s: ", noalias_str, arg_name);
354 fprintf(ar->f, "%s", noalias_str);
355 print_symbol(ar, &param_decl->data.param_decl.name);
356 fprintf(ar->f, ": ");
285357 }
286358 render_node(ar, param_decl->data.param_decl.type);
287359
......@@ -345,9 +417,10 @@ static void render_node(AstRender *ar, AstNode *node) {
345417 {
346418 const char *pub_str = visib_mod_string(node->data.variable_declaration.top_level_decl.visib_mod);
347419 const char *extern_str = extern_string(node->data.variable_declaration.is_extern);
348 const char *var_name = buf_ptr(&node->data.variable_declaration.symbol);
349420 const char *const_or_var = const_or_var_string(node->data.variable_declaration.is_const);
350 fprintf(ar->f, "%s%s%s %s", pub_str, extern_str, const_or_var, var_name);
421 fprintf(ar->f, "%s%s%s ", pub_str, extern_str, const_or_var);
422 print_symbol(ar, &node->data.variable_declaration.symbol);
423
351424 if (node->data.variable_declaration.type) {
352425 fprintf(ar->f, ": ");
353426 render_node(ar, node->data.variable_declaration.type);
......@@ -495,9 +568,8 @@ static void render_node(AstRender *ar, AstNode *node) {
495568 for (int field_i = 0; field_i < node->data.struct_decl.fields.length; field_i += 1) {
496569 AstNode *field_node = node->data.struct_decl.fields.at(field_i);
497570 assert(field_node->type == NodeTypeStructField);
498 const char *field_name = buf_ptr(&field_node->data.struct_field.name);
499571 print_indent(ar);
500 fprintf(ar->f, "%s", field_name);
572 print_symbol(ar, &field_node->data.struct_field.name);
501573 if (!is_node_void(field_node->data.struct_field.type)) {
502574 fprintf(ar->f, ": ");
503575 render_node(ar, field_node->data.struct_field.type);
src/parseh.cpp-5
......@@ -11,7 +11,6 @@
1111#include "error.hpp"
1212#include "parser.hpp"
1313#include "all_types.hpp"
14#include "tokenizer.hpp"
1514#include "c_tokenizer.hpp"
1615#include "analyze.hpp"
1716
......@@ -1265,10 +1264,6 @@ static void render_macros(Context *c) {
12651264}
12661265
12671266static void process_macro(Context *c, CTokenize *ctok, Buf *name, const char *char_ptr) {
1268 if (is_zig_keyword(name)) {
1269 return;
1270 }
1271
12721267 tokenize_c_macro(ctok, (const uint8_t *)char_ptr);
12731268
12741269 if (ctok->error) {
src/parser.cpp+97-78
......@@ -87,10 +87,6 @@ static AstNode *ast_create_void_type_node(ParseContext *pc, Token *token) {
8787 return node;
8888}
8989
90static void ast_buf_from_token(ParseContext *pc, Token *token, Buf *buf) {
91 buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
92}
93
9490static void parse_asm_template(ParseContext *pc, AstNode *node) {
9591 Buf *asm_template = &node->data.asm_expr.asm_template;
9692
......@@ -277,6 +273,8 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
277273 // detect c string literal
278274
279275 enum State {
276 StatePre,
277 StateSkipQuot,
280278 StateStart,
281279 StateEscape,
282280 StateHex1,
......@@ -285,90 +283,100 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
285283
286284 buf_resize(buf, 0);
287285
288 State state = StateStart;
289 bool skip_quote;
286 State state = StatePre;
290287 SrcPos pos = {token->start_line, token->start_column};
291288 int hex_value = 0;
292289 for (int i = token->start_pos; i < token->end_pos - 1; i += 1) {
293290 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);
294291
295 if (i == token->start_pos) {
296 skip_quote = (c == 'c');
297 if (out_c_str) {
298 *out_c_str = skip_quote;
299 } else if (skip_quote) {
300 ast_error(pc, token, "C string literal not allowed here");
301 }
302 } else if (skip_quote) {
303 skip_quote = false;
304 } else {
305 switch (state) {
306 case StateStart:
307 if (c == '\\') {
308 state = StateEscape;
309 } else {
310 buf_append_char(buf, c);
311 if (offset_map) offset_map->append(pos);
312 }
313 break;
314 case StateEscape:
315 switch (c) {
316 case '\\':
317 buf_append_char(buf, '\\');
318 if (offset_map) offset_map->append(pos);
319 state = StateStart;
320 break;
321 case 'r':
322 buf_append_char(buf, '\r');
323 if (offset_map) offset_map->append(pos);
324 state = StateStart;
325 break;
326 case 'n':
327 buf_append_char(buf, '\n');
328 if (offset_map) offset_map->append(pos);
329 state = StateStart;
330 break;
331 case 't':
332 buf_append_char(buf, '\t');
333 if (offset_map) offset_map->append(pos);
334 state = StateStart;
335 break;
336 case '"':
337 buf_append_char(buf, '"');
338 if (offset_map) offset_map->append(pos);
339 state = StateStart;
340 break;
341 case 'x':
342 state = StateHex1;
343 break;
344 default:
345 ast_error(pc, token, "invalid escape character");
346 break;
347 }
348 break;
349 case StateHex1:
350 {
351 int hex_digit = get_hex_digit(c);
352 if (hex_digit == -1) {
353 ast_error(pc, token, "invalid hex digit: '%c'", c);
354 }
355 hex_value = hex_digit * 16;
356 state = StateHex2;
292 switch (state) {
293 case StatePre:
294 switch (c) {
295 case '@':
296 state = StateSkipQuot;
357297 break;
358 }
359 case StateHex2:
360 {
361 int hex_digit = get_hex_digit(c);
362 if (hex_digit == -1) {
363 ast_error(pc, token, "invalid hex digit: '%c'", c);
298 case 'c':
299 if (out_c_str) {
300 *out_c_str = true;
301 } else {
302 ast_error(pc, token, "C string literal not allowed here");
364303 }
365 hex_value += hex_digit;
366 assert(hex_value >= 0 && hex_value <= 255);
367 buf_append_char(buf, hex_value);
304 state = StateSkipQuot;
305 break;
306 case '"':
368307 state = StateStart;
369308 break;
309 default:
310 ast_error(pc, token, "invalid string character");
311 }
312 break;
313 case StateSkipQuot:
314 state = StateStart;
315 break;
316 case StateStart:
317 if (c == '\\') {
318 state = StateEscape;
319 } else {
320 buf_append_char(buf, c);
321 if (offset_map) offset_map->append(pos);
322 }
323 break;
324 case StateEscape:
325 switch (c) {
326 case '\\':
327 buf_append_char(buf, '\\');
328 if (offset_map) offset_map->append(pos);
329 state = StateStart;
330 break;
331 case 'r':
332 buf_append_char(buf, '\r');
333 if (offset_map) offset_map->append(pos);
334 state = StateStart;
335 break;
336 case 'n':
337 buf_append_char(buf, '\n');
338 if (offset_map) offset_map->append(pos);
339 state = StateStart;
340 break;
341 case 't':
342 buf_append_char(buf, '\t');
343 if (offset_map) offset_map->append(pos);
344 state = StateStart;
345 break;
346 case '"':
347 buf_append_char(buf, '"');
348 if (offset_map) offset_map->append(pos);
349 state = StateStart;
350 break;
351 case 'x':
352 state = StateHex1;
353 break;
354 default:
355 ast_error(pc, token, "invalid escape character");
356 }
357 break;
358 case StateHex1:
359 {
360 int hex_digit = get_hex_digit(c);
361 if (hex_digit == -1) {
362 ast_error(pc, token, "invalid hex digit: '%c'", c);
370363 }
371 }
364 hex_value = hex_digit * 16;
365 state = StateHex2;
366 break;
367 }
368 case StateHex2:
369 {
370 int hex_digit = get_hex_digit(c);
371 if (hex_digit == -1) {
372 ast_error(pc, token, "invalid hex digit: '%c'", c);
373 }
374 hex_value += hex_digit;
375 assert(hex_value >= 0 && hex_value <= 255);
376 buf_append_char(buf, hex_value);
377 state = StateStart;
378 break;
379 }
372380 }
373381 if (c == '\n') {
374382 pos.line += 1;
......@@ -381,6 +389,17 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
381389 if (offset_map) offset_map->append(pos);
382390}
383391
392static void ast_buf_from_token(ParseContext *pc, Token *token, Buf *buf) {
393 uint8_t *first_char = (uint8_t *)buf_ptr(pc->buf) + token->start_pos;
394 bool at_sign = *first_char == '@';
395 if (at_sign) {
396 parse_string_literal(pc, token, buf, nullptr, nullptr);
397 } else {
398 buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
399 }
400}
401
402
384403static unsigned long long parse_int_digits(ParseContext *pc, int digits_start, int digits_end, int radix,
385404 int skip_index, bool *overflow)
386405{
src/tokenizer.cpp+16-1
......@@ -159,6 +159,7 @@ enum TokenizeState {
159159 TokenizeStateSawDot,
160160 TokenizeStateSawDotDot,
161161 TokenizeStateSawQuestionMark,
162 TokenizeStateSawAtSign,
162163 TokenizeStateError,
163164};
164165
......@@ -429,7 +430,7 @@ void tokenize(Buf *buf, Tokenization *out) {
429430 break;
430431 case '@':
431432 begin_token(&t, TokenIdAtSign);
432 end_token(&t);
433 t.state = TokenizeStateSawAtSign;
433434 break;
434435 case '-':
435436 begin_token(&t, TokenIdDash);
......@@ -858,6 +859,19 @@ void tokenize(Buf *buf, Tokenization *out) {
858859 continue;
859860 }
860861 break;
862 case TokenizeStateSawAtSign:
863 switch (c) {
864 case '"':
865 t.cur_tok->id = TokenIdSymbol;
866 t.state = TokenizeStateString;
867 break;
868 default:
869 t.pos -= 1;
870 end_token(&t);
871 t.state = TokenizeStateStart;
872 continue;
873 }
874 break;
861875 case TokenizeStateFirstR:
862876 switch (c) {
863877 case '"':
......@@ -1131,6 +1145,7 @@ void tokenize(Buf *buf, Tokenization *out) {
11311145 case TokenizeStateSawGreaterThanGreaterThan:
11321146 case TokenizeStateSawDot:
11331147 case TokenizeStateSawQuestionMark:
1148 case TokenizeStateSawAtSign:
11341149 end_token(&t);
11351150 break;
11361151 case TokenizeStateSawDotDot:
test/run_tests.cpp+8
......@@ -1394,6 +1394,14 @@ void foo(void (__cdecl *fn_ptr)(void));
13941394 add_parseh_case("comment after integer literal", R"SOURCE(
13951395#define SDL_INIT_VIDEO 0x00000020 /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
13961396 )SOURCE", 1, "pub const SDL_INIT_VIDEO = 32;");
1397
1398 add_parseh_case("zig keywords in C code", R"SOURCE(
1399struct type {
1400 int defer;
1401};
1402 )SOURCE", 2, R"(export struct struct_type {
1403 @"defer": c_int,
1404})", R"(pub const @"type" = struct_type;)");
13971405}
13981406
13991407static void run_self_hosted_test(void) {
test/self_hosted.zig+4
......@@ -1295,3 +1295,7 @@ struct EmptyStruct {
12951295 #static_eval_enable(false)
12961296 fn method(es: EmptyStruct) -> i32 { 1234 }
12971297}
1298
1299
1300#attribute("test")
1301fn @"weird function name"() { }