authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-27 00:40:26-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-27 00:40:26-07:00
log821907317eb77a96dc53adf20ac705b4501e2ab8
tree742da51aa2306895b1605877904d2763402989e2
parent4060ae93fb14b333166ff9d8ba880ff128989d31

support C-style comments, plus nesting

4 files changed, 124 insertions(+), 14 deletions(-)

README.md+1-1
...@@ -31,7 +31,7 @@ readable, safe, optimal, and concise code to solve any computing problem....@@ -31,7 +31,7 @@ readable, safe, optimal, and concise code to solve any computing problem.
3131
32## Roadmap32## Roadmap
3333
34 * C style comments.34 * empty function and return with no expression
35 * Simple .so library35 * Simple .so library
36 * Multiple files36 * Multiple files
37 * figure out integers37 * figure out integers
src/tokenizer.cpp+100-3
...@@ -87,6 +87,19 @@...@@ -87,6 +87,19 @@
87 case DIGIT: \87 case DIGIT: \
88 case '_'88 case '_'
8989
90enum TokenizeState {
91 TokenizeStateStart,
92 TokenizeStateSymbol,
93 TokenizeStateNumber,
94 TokenizeStateString,
95 TokenizeStateSawDash,
96 TokenizeStateSawSlash,
97 TokenizeStateLineComment,
98 TokenizeStateMultiLineComment,
99 TokenizeStateMultiLineCommentSlash,
100 TokenizeStateMultiLineCommentStar,
101};
102
90103
91struct Tokenize {104struct Tokenize {
92 Buf *buf;105 Buf *buf;
...@@ -96,6 +109,7 @@ struct Tokenize {...@@ -96,6 +109,7 @@ struct Tokenize {
96 int line;109 int line;
97 int column;110 int column;
98 Token *cur_tok;111 Token *cur_tok;
112 int multi_line_comment_count;
99};113};
100114
101__attribute__ ((format (printf, 2, 3)))115__attribute__ ((format (printf, 2, 3)))
...@@ -222,8 +236,78 @@ ZigList<Token> *tokenize(Buf *buf) {...@@ -222,8 +236,78 @@ ZigList<Token> *tokenize(Buf *buf) {
222 begin_token(&t, TokenIdNumberSign);236 begin_token(&t, TokenIdNumberSign);
223 end_token(&t);237 end_token(&t);
224 break;238 break;
239 case '/':
240 t.state = TokenizeStateSawSlash;
241 break;
242 default:
243 tokenize_error(&t, "invalid character: '%c'", c);
244 }
245 break;
246 case TokenizeStateSawSlash:
247 switch (c) {
248 case '/':
249 t.state = TokenizeStateLineComment;
250 break;
251 case '*':
252 t.state = TokenizeStateMultiLineComment;
253 t.multi_line_comment_count = 1;
254 break;
225 default:255 default:
226 tokenize_error(&t, "invalid character: '%c'", c);256 tokenize_error(&t, "invalid character: '%c'", c);
257 break;
258 }
259 break;
260 case TokenizeStateLineComment:
261 switch (c) {
262 case '\n':
263 t.state = TokenizeStateStart;
264 break;
265 default:
266 // do nothing
267 break;
268 }
269 break;
270 case TokenizeStateMultiLineComment:
271 switch (c) {
272 case '*':
273 t.state = TokenizeStateMultiLineCommentStar;
274 break;
275 case '/':
276 t.state = TokenizeStateMultiLineCommentSlash;
277 break;
278 default:
279 // do nothing
280 break;
281 }
282 break;
283 case TokenizeStateMultiLineCommentSlash:
284 switch (c) {
285 case '*':
286 t.state = TokenizeStateMultiLineComment;
287 t.multi_line_comment_count += 1;
288 break;
289 case '/':
290 break;
291 default:
292 t.state = TokenizeStateMultiLineComment;
293 break;
294 }
295 break;
296 case TokenizeStateMultiLineCommentStar:
297 switch (c) {
298 case '/':
299 t.multi_line_comment_count -= 1;
300 if (t.multi_line_comment_count == 0) {
301 t.state = TokenizeStateStart;
302 } else {
303 t.state = TokenizeStateMultiLineComment;
304 }
305 break;
306 case '*':
307 break;
308 default:
309 t.state = TokenizeStateMultiLineComment;
310 break;
227 }311 }
228 break;312 break;
229 case TokenizeStateSymbol:313 case TokenizeStateSymbol:
...@@ -295,7 +379,18 @@ ZigList<Token> *tokenize(Buf *buf) {...@@ -295,7 +379,18 @@ ZigList<Token> *tokenize(Buf *buf) {
295 case TokenizeStateSawDash:379 case TokenizeStateSawDash:
296 end_token(&t);380 end_token(&t);
297 break;381 break;
382 case TokenizeStateSawSlash:
383 tokenize_error(&t, "unexpected EOF");
384 break;
385 case TokenizeStateLineComment:
386 break;
387 case TokenizeStateMultiLineComment:
388 case TokenizeStateMultiLineCommentSlash:
389 case TokenizeStateMultiLineCommentStar:
390 tokenize_error(&t, "unterminated multi-line comment");
391 break;
298 }392 }
393 t.pos = -1;
299 begin_token(&t, TokenIdEof);394 begin_token(&t, TokenIdEof);
300 end_token(&t);395 end_token(&t);
301 assert(!t.cur_tok);396 assert(!t.cur_tok);
...@@ -333,9 +428,11 @@ static const char * token_name(Token *token) {...@@ -333,9 +428,11 @@ static const char * token_name(Token *token) {
333void print_tokens(Buf *buf, ZigList<Token> *tokens) {428void print_tokens(Buf *buf, ZigList<Token> *tokens) {
334 for (int i = 0; i < tokens->length; i += 1) {429 for (int i = 0; i < tokens->length; i += 1) {
335 Token *token = &tokens->at(i);430 Token *token = &tokens->at(i);
336 printf("%s ", token_name(token));431 fprintf(stderr, "%s ", token_name(token));
337 fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stdout);432 if (token->start_pos >= 0) {
338 printf("\n");433 fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stderr);
434 }
435 fprintf(stderr, "\n");
339 }436 }
340}437}
341438
src/tokenizer.hpp-8
...@@ -43,14 +43,6 @@ struct Token {...@@ -43,14 +43,6 @@ struct Token {
43 int start_column;43 int start_column;
44};44};
4545
46enum TokenizeState {
47 TokenizeStateStart,
48 TokenizeStateSymbol,
49 TokenizeStateNumber,
50 TokenizeStateString,
51 TokenizeStateSawDash,
52};
53
54ZigList<Token> *tokenize(Buf *buf);46ZigList<Token> *tokenize(Buf *buf);
5547
56void print_tokens(Buf *buf, ZigList<Token> *tokens);48void print_tokens(Buf *buf, ZigList<Token> *tokens);
test/standalone.cpp+23-2
...@@ -75,6 +75,26 @@ static void add_all_test_cases(void) {...@@ -75,6 +75,26 @@ static void add_all_test_cases(void) {
75 exit(0);75 exit(0);
76 }76 }
77 )SOURCE", "OK\n");77 )SOURCE", "OK\n");
78
79 add_simple_case("comments", R"SOURCE(
80 #link("c")
81 extern {
82 fn puts(s: *mut u8) -> i32;
83 fn exit(code: i32) -> unreachable;
84 }
85
86 /**
87 * multi line doc comment
88 */
89 fn another_function() -> i32 { return 0; }
90
91 /// this is a documentation comment
92 /// doc comment line 2
93 fn _start() -> unreachable {
94 puts(/* mid-line comment /* nested */ */ "OK");
95 exit(0);
96 }
97 )SOURCE", "OK\n");
78}98}
7999
80static void run_test(TestCase *test_case) {100static void run_test(TestCase *test_case) {
...@@ -83,11 +103,12 @@ static void run_test(TestCase *test_case) {...@@ -83,11 +103,12 @@ static void run_test(TestCase *test_case) {
83 Buf zig_stderr = BUF_INIT;103 Buf zig_stderr = BUF_INIT;
84 Buf zig_stdout = BUF_INIT;104 Buf zig_stdout = BUF_INIT;
85 int return_code;105 int return_code;
86 os_exec_process("./zig", test_case->compiler_args, &return_code, &zig_stderr, &zig_stdout);106 static const char *zig_exe = "./zig";
107 os_exec_process(zig_exe, test_case->compiler_args, &return_code, &zig_stderr, &zig_stdout);
87108
88 if (return_code != 0) {109 if (return_code != 0) {
89 printf("\nCompile failed with return code %d:\n", return_code);110 printf("\nCompile failed with return code %d:\n", return_code);
90 printf("zig");111 printf("%s", zig_exe);
91 for (int i = 0; i < test_case->compiler_args.length; i += 1) {112 for (int i = 0; i < test_case->compiler_args.length; i += 1) {
92 printf(" %s", test_case->compiler_args.at(i));113 printf(" %s", test_case->compiler_args.at(i));
93 }114 }