authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-22 15:24:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2016-04-22 15:24:18-07:00
log8187396f640d2a56e0d56c7d199074b4590e49eb
tree5dbbeeba42680a30c43ccd1d1be3be89925d73d3
parent35362f8137b2c5109e6bc39cb12048c016b5b580

add syntax to allow symbols to have arbitrary strings as names

6 files changed, 208 insertions(+), 95 deletions(-)

src/ast_render.cpp+83-11
...@@ -239,10 +239,80 @@ static bool is_node_void(AstNode *node) {...@@ -239,10 +239,80 @@ static bool is_node_void(AstNode *node) {
239 return false;239 return false;
240}240}
241241
242static bool is_printable(uint8_t c) {242static bool is_alpha_under(uint8_t c) {
243 return (c >= 'a' && c <= 'z') ||243 return (c >= 'a' && c <= 'z') ||
244 (c >= 'A' && c <= 'A') ||244 (c >= 'A' && c <= 'Z') || c == '_';
245 (c >= '0' && c <= '9');245}
246
247static bool is_digit(uint8_t c) {
248 return (c >= '0' && c <= '9');
249}
250
251static bool is_printable(uint8_t c) {
252 return is_alpha_under(c) || is_digit(c);
253}
254
255static void string_literal_escape(Buf *source, Buf *dest) {
256 buf_resize(dest, 0);
257 for (int i = 0; i < buf_len(source); i += 1) {
258 uint8_t c = *((uint8_t*)buf_ptr(source) + i);
259 if (is_printable(c)) {
260 buf_append_char(dest, c);
261 } else if (c == '\'') {
262 buf_append_str(dest, "\\'");
263 } else if (c == '"') {
264 buf_append_str(dest, "\\\"");
265 } else if (c == '\\') {
266 buf_append_str(dest, "\\\\");
267 } else if (c == '\a') {
268 buf_append_str(dest, "\\a");
269 } else if (c == '\b') {
270 buf_append_str(dest, "\\b");
271 } else if (c == '\f') {
272 buf_append_str(dest, "\\f");
273 } else if (c == '\n') {
274 buf_append_str(dest, "\\n");
275 } else if (c == '\r') {
276 buf_append_str(dest, "\\r");
277 } else if (c == '\t') {
278 buf_append_str(dest, "\\t");
279 } else if (c == '\v') {
280 buf_append_str(dest, "\\v");
281 } else {
282 buf_appendf(dest, "\\x%x", (int)c);
283 }
284 }
285}
286
287static bool is_valid_bare_symbol(Buf *symbol) {
288 if (buf_len(symbol) == 0) {
289 return false;
290 }
291 uint8_t first_char = *buf_ptr(symbol);
292 if (!is_alpha_under(first_char)) {
293 return false;
294 }
295 for (int i = 1; i < buf_len(symbol); i += 1) {
296 uint8_t c = *((uint8_t*)buf_ptr(symbol) + i);
297 if (!is_alpha_under(c) && !is_digit(c)) {
298 return false;
299 }
300 }
301 return true;
302}
303
304static void print_symbol(AstRender *ar, Buf *symbol) {
305 if (is_zig_keyword(symbol)) {
306 fprintf(ar->f, "@\"%s\"", buf_ptr(symbol));
307 return;
308 }
309 if (is_valid_bare_symbol(symbol)) {
310 fprintf(ar->f, "%s", buf_ptr(symbol));
311 return;
312 }
313 Buf escaped = BUF_INIT;
314 string_literal_escape(symbol, &escaped);
315 fprintf(ar->f, "@\"%s\"", buf_ptr(&escaped));
246}316}
247317
248static void render_node(AstRender *ar, AstNode *node) {318static void render_node(AstRender *ar, AstNode *node) {
...@@ -268,20 +338,22 @@ static void render_node(AstRender *ar, AstNode *node) {...@@ -268,20 +338,22 @@ static void render_node(AstRender *ar, AstNode *node) {
268 break;338 break;
269 case NodeTypeFnProto:339 case NodeTypeFnProto:
270 {340 {
271 const char *fn_name = buf_ptr(&node->data.fn_proto.name);
272 const char *pub_str = visib_mod_string(node->data.fn_proto.top_level_decl.visib_mod);341 const char *pub_str = visib_mod_string(node->data.fn_proto.top_level_decl.visib_mod);
273 const char *extern_str = extern_string(node->data.fn_proto.is_extern);342 const char *extern_str = extern_string(node->data.fn_proto.is_extern);
274 const char *inline_str = inline_string(node->data.fn_proto.is_inline);343 const char *inline_str = inline_string(node->data.fn_proto.is_inline);
275 fprintf(ar->f, "%s%s%sfn %s(", pub_str, inline_str, extern_str, fn_name);344 fprintf(ar->f, "%s%s%sfn ", pub_str, inline_str, extern_str);
345 print_symbol(ar, &node->data.fn_proto.name);
346 fprintf(ar->f, "(");
276 int arg_count = node->data.fn_proto.params.length;347 int arg_count = node->data.fn_proto.params.length;
277 bool is_var_args = node->data.fn_proto.is_var_args;348 bool is_var_args = node->data.fn_proto.is_var_args;
278 for (int arg_i = 0; arg_i < arg_count; arg_i += 1) {349 for (int arg_i = 0; arg_i < arg_count; arg_i += 1) {
279 AstNode *param_decl = node->data.fn_proto.params.at(arg_i);350 AstNode *param_decl = node->data.fn_proto.params.at(arg_i);
280 assert(param_decl->type == NodeTypeParamDecl);351 assert(param_decl->type == NodeTypeParamDecl);
281 const char *arg_name = buf_ptr(&param_decl->data.param_decl.name);
282 if (buf_len(&param_decl->data.param_decl.name) > 0) {352 if (buf_len(&param_decl->data.param_decl.name) > 0) {
283 const char *noalias_str = param_decl->data.param_decl.is_noalias ? "noalias " : "";353 const char *noalias_str = param_decl->data.param_decl.is_noalias ? "noalias " : "";
284 fprintf(ar->f, "%s%s: ", noalias_str, arg_name);354 fprintf(ar->f, "%s", noalias_str);
355 print_symbol(ar, &param_decl->data.param_decl.name);
356 fprintf(ar->f, ": ");
285 }357 }
286 render_node(ar, param_decl->data.param_decl.type);358 render_node(ar, param_decl->data.param_decl.type);
287359
...@@ -345,9 +417,10 @@ static void render_node(AstRender *ar, AstNode *node) {...@@ -345,9 +417,10 @@ static void render_node(AstRender *ar, AstNode *node) {
345 {417 {
346 const char *pub_str = visib_mod_string(node->data.variable_declaration.top_level_decl.visib_mod);418 const char *pub_str = visib_mod_string(node->data.variable_declaration.top_level_decl.visib_mod);
347 const char *extern_str = extern_string(node->data.variable_declaration.is_extern);419 const char *extern_str = extern_string(node->data.variable_declaration.is_extern);
348 const char *var_name = buf_ptr(&node->data.variable_declaration.symbol);
349 const char *const_or_var = const_or_var_string(node->data.variable_declaration.is_const);420 const char *const_or_var = const_or_var_string(node->data.variable_declaration.is_const);
350 fprintf(ar->f, "%s%s%s %s", pub_str, extern_str, const_or_var, var_name);421 fprintf(ar->f, "%s%s%s ", pub_str, extern_str, const_or_var);
422 print_symbol(ar, &node->data.variable_declaration.symbol);
423
351 if (node->data.variable_declaration.type) {424 if (node->data.variable_declaration.type) {
352 fprintf(ar->f, ": ");425 fprintf(ar->f, ": ");
353 render_node(ar, node->data.variable_declaration.type);426 render_node(ar, node->data.variable_declaration.type);
...@@ -495,9 +568,8 @@ static void render_node(AstRender *ar, AstNode *node) {...@@ -495,9 +568,8 @@ static void render_node(AstRender *ar, AstNode *node) {
495 for (int field_i = 0; field_i < node->data.struct_decl.fields.length; field_i += 1) {568 for (int field_i = 0; field_i < node->data.struct_decl.fields.length; field_i += 1) {
496 AstNode *field_node = node->data.struct_decl.fields.at(field_i);569 AstNode *field_node = node->data.struct_decl.fields.at(field_i);
497 assert(field_node->type == NodeTypeStructField);570 assert(field_node->type == NodeTypeStructField);
498 const char *field_name = buf_ptr(&field_node->data.struct_field.name);
499 print_indent(ar);571 print_indent(ar);
500 fprintf(ar->f, "%s", field_name);572 print_symbol(ar, &field_node->data.struct_field.name);
501 if (!is_node_void(field_node->data.struct_field.type)) {573 if (!is_node_void(field_node->data.struct_field.type)) {
502 fprintf(ar->f, ": ");574 fprintf(ar->f, ": ");
503 render_node(ar, field_node->data.struct_field.type);575 render_node(ar, field_node->data.struct_field.type);
src/parseh.cpp-5
...@@ -11,7 +11,6 @@...@@ -11,7 +11,6 @@
11#include "error.hpp"11#include "error.hpp"
12#include "parser.hpp"12#include "parser.hpp"
13#include "all_types.hpp"13#include "all_types.hpp"
14#include "tokenizer.hpp"
15#include "c_tokenizer.hpp"14#include "c_tokenizer.hpp"
16#include "analyze.hpp"15#include "analyze.hpp"
1716
...@@ -1265,10 +1264,6 @@ static void render_macros(Context *c) {...@@ -1265,10 +1264,6 @@ static void render_macros(Context *c) {
1265}1264}
12661265
1267static void process_macro(Context *c, CTokenize *ctok, Buf *name, const char *char_ptr) {1266static void process_macro(Context *c, CTokenize *ctok, Buf *name, const char *char_ptr) {
1268 if (is_zig_keyword(name)) {
1269 return;
1270 }
1271
1272 tokenize_c_macro(ctok, (const uint8_t *)char_ptr);1267 tokenize_c_macro(ctok, (const uint8_t *)char_ptr);
12731268
1274 if (ctok->error) {1269 if (ctok->error) {
src/parser.cpp+97-78
...@@ -87,10 +87,6 @@ static AstNode *ast_create_void_type_node(ParseContext *pc, Token *token) {...@@ -87,10 +87,6 @@ static AstNode *ast_create_void_type_node(ParseContext *pc, Token *token) {
87 return node;87 return node;
88}88}
8989
90static void ast_buf_from_token(ParseContext *pc, Token *token, Buf *buf) {
91 buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
92}
93
94static void parse_asm_template(ParseContext *pc, AstNode *node) {90static void parse_asm_template(ParseContext *pc, AstNode *node) {
95 Buf *asm_template = &node->data.asm_expr.asm_template;91 Buf *asm_template = &node->data.asm_expr.asm_template;
9692
...@@ -277,6 +273,8 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool...@@ -277,6 +273,8 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
277 // detect c string literal273 // detect c string literal
278274
279 enum State {275 enum State {
276 StatePre,
277 StateSkipQuot,
280 StateStart,278 StateStart,
281 StateEscape,279 StateEscape,
282 StateHex1,280 StateHex1,
...@@ -285,90 +283,100 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool...@@ -285,90 +283,100 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
285283
286 buf_resize(buf, 0);284 buf_resize(buf, 0);
287285
288 State state = StateStart;286 State state = StatePre;
289 bool skip_quote;
290 SrcPos pos = {token->start_line, token->start_column};287 SrcPos pos = {token->start_line, token->start_column};
291 int hex_value = 0;288 int hex_value = 0;
292 for (int i = token->start_pos; i < token->end_pos - 1; i += 1) {289 for (int i = token->start_pos; i < token->end_pos - 1; i += 1) {
293 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);290 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);
294291
295 if (i == token->start_pos) {292 switch (state) {
296 skip_quote = (c == 'c');293 case StatePre:
297 if (out_c_str) {294 switch (c) {
298 *out_c_str = skip_quote;295 case '@':
299 } else if (skip_quote) {296 state = StateSkipQuot;
300 ast_error(pc, token, "C string literal not allowed here");
301 }
302 } else if (skip_quote) {
303 skip_quote = false;
304 } else {
305 switch (state) {
306 case StateStart:
307 if (c == '\\') {
308 state = StateEscape;
309 } else {
310 buf_append_char(buf, c);
311 if (offset_map) offset_map->append(pos);
312 }
313 break;
314 case StateEscape:
315 switch (c) {
316 case '\\':
317 buf_append_char(buf, '\\');
318 if (offset_map) offset_map->append(pos);
319 state = StateStart;
320 break;
321 case 'r':
322 buf_append_char(buf, '\r');
323 if (offset_map) offset_map->append(pos);
324 state = StateStart;
325 break;
326 case 'n':
327 buf_append_char(buf, '\n');
328 if (offset_map) offset_map->append(pos);
329 state = StateStart;
330 break;
331 case 't':
332 buf_append_char(buf, '\t');
333 if (offset_map) offset_map->append(pos);
334 state = StateStart;
335 break;
336 case '"':
337 buf_append_char(buf, '"');
338 if (offset_map) offset_map->append(pos);
339 state = StateStart;
340 break;
341 case 'x':
342 state = StateHex1;
343 break;
344 default:
345 ast_error(pc, token, "invalid escape character");
346 break;
347 }
348 break;
349 case StateHex1:
350 {
351 int hex_digit = get_hex_digit(c);
352 if (hex_digit == -1) {
353 ast_error(pc, token, "invalid hex digit: '%c'", c);
354 }
355 hex_value = hex_digit * 16;
356 state = StateHex2;
357 break;297 break;
358 }298 case 'c':
359 case StateHex2:299 if (out_c_str) {
360 {300 *out_c_str = true;
361 int hex_digit = get_hex_digit(c);301 } else {
362 if (hex_digit == -1) {302 ast_error(pc, token, "C string literal not allowed here");
363 ast_error(pc, token, "invalid hex digit: '%c'", c);
364 }303 }
365 hex_value += hex_digit;304 state = StateSkipQuot;
366 assert(hex_value >= 0 && hex_value <= 255);305 break;
367 buf_append_char(buf, hex_value);306 case '"':
368 state = StateStart;307 state = StateStart;
369 break;308 break;
309 default:
310 ast_error(pc, token, "invalid string character");
311 }
312 break;
313 case StateSkipQuot:
314 state = StateStart;
315 break;
316 case StateStart:
317 if (c == '\\') {
318 state = StateEscape;
319 } else {
320 buf_append_char(buf, c);
321 if (offset_map) offset_map->append(pos);
322 }
323 break;
324 case StateEscape:
325 switch (c) {
326 case '\\':
327 buf_append_char(buf, '\\');
328 if (offset_map) offset_map->append(pos);
329 state = StateStart;
330 break;
331 case 'r':
332 buf_append_char(buf, '\r');
333 if (offset_map) offset_map->append(pos);
334 state = StateStart;
335 break;
336 case 'n':
337 buf_append_char(buf, '\n');
338 if (offset_map) offset_map->append(pos);
339 state = StateStart;
340 break;
341 case 't':
342 buf_append_char(buf, '\t');
343 if (offset_map) offset_map->append(pos);
344 state = StateStart;
345 break;
346 case '"':
347 buf_append_char(buf, '"');
348 if (offset_map) offset_map->append(pos);
349 state = StateStart;
350 break;
351 case 'x':
352 state = StateHex1;
353 break;
354 default:
355 ast_error(pc, token, "invalid escape character");
356 }
357 break;
358 case StateHex1:
359 {
360 int hex_digit = get_hex_digit(c);
361 if (hex_digit == -1) {
362 ast_error(pc, token, "invalid hex digit: '%c'", c);
370 }363 }
371 }364 hex_value = hex_digit * 16;
365 state = StateHex2;
366 break;
367 }
368 case StateHex2:
369 {
370 int hex_digit = get_hex_digit(c);
371 if (hex_digit == -1) {
372 ast_error(pc, token, "invalid hex digit: '%c'", c);
373 }
374 hex_value += hex_digit;
375 assert(hex_value >= 0 && hex_value <= 255);
376 buf_append_char(buf, hex_value);
377 state = StateStart;
378 break;
379 }
372 }380 }
373 if (c == '\n') {381 if (c == '\n') {
374 pos.line += 1;382 pos.line += 1;
...@@ -381,6 +389,17 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool...@@ -381,6 +389,17 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
381 if (offset_map) offset_map->append(pos);389 if (offset_map) offset_map->append(pos);
382}390}
383391
392static void ast_buf_from_token(ParseContext *pc, Token *token, Buf *buf) {
393 uint8_t *first_char = (uint8_t *)buf_ptr(pc->buf) + token->start_pos;
394 bool at_sign = *first_char == '@';
395 if (at_sign) {
396 parse_string_literal(pc, token, buf, nullptr, nullptr);
397 } else {
398 buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
399 }
400}
401
402
384static unsigned long long parse_int_digits(ParseContext *pc, int digits_start, int digits_end, int radix,403static unsigned long long parse_int_digits(ParseContext *pc, int digits_start, int digits_end, int radix,
385 int skip_index, bool *overflow)404 int skip_index, bool *overflow)
386{405{
src/tokenizer.cpp+16-1
...@@ -159,6 +159,7 @@ enum TokenizeState {...@@ -159,6 +159,7 @@ enum TokenizeState {
159 TokenizeStateSawDot,159 TokenizeStateSawDot,
160 TokenizeStateSawDotDot,160 TokenizeStateSawDotDot,
161 TokenizeStateSawQuestionMark,161 TokenizeStateSawQuestionMark,
162 TokenizeStateSawAtSign,
162 TokenizeStateError,163 TokenizeStateError,
163};164};
164165
...@@ -429,7 +430,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -429,7 +430,7 @@ void tokenize(Buf *buf, Tokenization *out) {
429 break;430 break;
430 case '@':431 case '@':
431 begin_token(&t, TokenIdAtSign);432 begin_token(&t, TokenIdAtSign);
432 end_token(&t);433 t.state = TokenizeStateSawAtSign;
433 break;434 break;
434 case '-':435 case '-':
435 begin_token(&t, TokenIdDash);436 begin_token(&t, TokenIdDash);
...@@ -858,6 +859,19 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -858,6 +859,19 @@ void tokenize(Buf *buf, Tokenization *out) {
858 continue;859 continue;
859 }860 }
860 break;861 break;
862 case TokenizeStateSawAtSign:
863 switch (c) {
864 case '"':
865 t.cur_tok->id = TokenIdSymbol;
866 t.state = TokenizeStateString;
867 break;
868 default:
869 t.pos -= 1;
870 end_token(&t);
871 t.state = TokenizeStateStart;
872 continue;
873 }
874 break;
861 case TokenizeStateFirstR:875 case TokenizeStateFirstR:
862 switch (c) {876 switch (c) {
863 case '"':877 case '"':
...@@ -1131,6 +1145,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1131,6 +1145,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1131 case TokenizeStateSawGreaterThanGreaterThan:1145 case TokenizeStateSawGreaterThanGreaterThan:
1132 case TokenizeStateSawDot:1146 case TokenizeStateSawDot:
1133 case TokenizeStateSawQuestionMark:1147 case TokenizeStateSawQuestionMark:
1148 case TokenizeStateSawAtSign:
1134 end_token(&t);1149 end_token(&t);
1135 break;1150 break;
1136 case TokenizeStateSawDotDot:1151 case TokenizeStateSawDotDot:
test/run_tests.cpp+8
...@@ -1394,6 +1394,14 @@ void foo(void (__cdecl *fn_ptr)(void));...@@ -1394,6 +1394,14 @@ void foo(void (__cdecl *fn_ptr)(void));
1394 add_parseh_case("comment after integer literal", R"SOURCE(1394 add_parseh_case("comment after integer literal", R"SOURCE(
1395#define SDL_INIT_VIDEO 0x00000020 /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */1395#define SDL_INIT_VIDEO 0x00000020 /**< SDL_INIT_VIDEO implies SDL_INIT_EVENTS */
1396 )SOURCE", 1, "pub const SDL_INIT_VIDEO = 32;");1396 )SOURCE", 1, "pub const SDL_INIT_VIDEO = 32;");
1397
1398 add_parseh_case("zig keywords in C code", R"SOURCE(
1399struct type {
1400 int defer;
1401};
1402 )SOURCE", 2, R"(export struct struct_type {
1403 @"defer": c_int,
1404})", R"(pub const @"type" = struct_type;)");
1397}1405}
13981406
1399static void run_self_hosted_test(void) {1407static void run_self_hosted_test(void) {
test/self_hosted.zig+4
...@@ -1295,3 +1295,7 @@ struct EmptyStruct {...@@ -1295,3 +1295,7 @@ struct EmptyStruct {
1295 #static_eval_enable(false)1295 #static_eval_enable(false)
1296 fn method(es: EmptyStruct) -> i32 { 1234 }1296 fn method(es: EmptyStruct) -> i32 { 1234 }
1297}1297}
1298
1299
1300#attribute("test")
1301fn @"weird function name"() { }