authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-27 00:40:26-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-27 00:40:26-07:00
log821907317eb77a96dc53adf20ac705b4501e2ab8
tree742da51aa2306895b1605877904d2763402989e2
parent4060ae93fb14b333166ff9d8ba880ff128989d31

support C-style comments, plus nesting

4 files changed, 124 insertions(+), 14 deletions(-)

README.md+1-1
......@@ -31,7 +31,7 @@ readable, safe, optimal, and concise code to solve any computing problem.
3131
3232## Roadmap
3333
34 * C style comments.
34 * empty function and return with no expression
3535 * Simple .so library
3636 * Multiple files
3737 * figure out integers
src/tokenizer.cpp+100-3
......@@ -87,6 +87,19 @@
8787 case DIGIT: \
8888 case '_'
8989
90enum TokenizeState {
91 TokenizeStateStart,
92 TokenizeStateSymbol,
93 TokenizeStateNumber,
94 TokenizeStateString,
95 TokenizeStateSawDash,
96 TokenizeStateSawSlash,
97 TokenizeStateLineComment,
98 TokenizeStateMultiLineComment,
99 TokenizeStateMultiLineCommentSlash,
100 TokenizeStateMultiLineCommentStar,
101};
102
90103
91104struct Tokenize {
92105 Buf *buf;
......@@ -96,6 +109,7 @@ struct Tokenize {
96109 int line;
97110 int column;
98111 Token *cur_tok;
112 int multi_line_comment_count;
99113};
100114
101115__attribute__ ((format (printf, 2, 3)))
......@@ -222,8 +236,78 @@ ZigList<Token> *tokenize(Buf *buf) {
222236 begin_token(&t, TokenIdNumberSign);
223237 end_token(&t);
224238 break;
239 case '/':
240 t.state = TokenizeStateSawSlash;
241 break;
242 default:
243 tokenize_error(&t, "invalid character: '%c'", c);
244 }
245 break;
246 case TokenizeStateSawSlash:
247 switch (c) {
248 case '/':
249 t.state = TokenizeStateLineComment;
250 break;
251 case '*':
252 t.state = TokenizeStateMultiLineComment;
253 t.multi_line_comment_count = 1;
254 break;
225255 default:
226256 tokenize_error(&t, "invalid character: '%c'", c);
257 break;
258 }
259 break;
260 case TokenizeStateLineComment:
261 switch (c) {
262 case '\n':
263 t.state = TokenizeStateStart;
264 break;
265 default:
266 // do nothing
267 break;
268 }
269 break;
270 case TokenizeStateMultiLineComment:
271 switch (c) {
272 case '*':
273 t.state = TokenizeStateMultiLineCommentStar;
274 break;
275 case '/':
276 t.state = TokenizeStateMultiLineCommentSlash;
277 break;
278 default:
279 // do nothing
280 break;
281 }
282 break;
283 case TokenizeStateMultiLineCommentSlash:
284 switch (c) {
285 case '*':
286 t.state = TokenizeStateMultiLineComment;
287 t.multi_line_comment_count += 1;
288 break;
289 case '/':
290 break;
291 default:
292 t.state = TokenizeStateMultiLineComment;
293 break;
294 }
295 break;
296 case TokenizeStateMultiLineCommentStar:
297 switch (c) {
298 case '/':
299 t.multi_line_comment_count -= 1;
300 if (t.multi_line_comment_count == 0) {
301 t.state = TokenizeStateStart;
302 } else {
303 t.state = TokenizeStateMultiLineComment;
304 }
305 break;
306 case '*':
307 break;
308 default:
309 t.state = TokenizeStateMultiLineComment;
310 break;
227311 }
228312 break;
229313 case TokenizeStateSymbol:
......@@ -295,7 +379,18 @@ ZigList<Token> *tokenize(Buf *buf) {
295379 case TokenizeStateSawDash:
296380 end_token(&t);
297381 break;
382 case TokenizeStateSawSlash:
383 tokenize_error(&t, "unexpected EOF");
384 break;
385 case TokenizeStateLineComment:
386 break;
387 case TokenizeStateMultiLineComment:
388 case TokenizeStateMultiLineCommentSlash:
389 case TokenizeStateMultiLineCommentStar:
390 tokenize_error(&t, "unterminated multi-line comment");
391 break;
298392 }
393 t.pos = -1;
299394 begin_token(&t, TokenIdEof);
300395 end_token(&t);
301396 assert(!t.cur_tok);
......@@ -333,9 +428,11 @@ static const char * token_name(Token *token) {
333428void print_tokens(Buf *buf, ZigList<Token> *tokens) {
334429 for (int i = 0; i < tokens->length; i += 1) {
335430 Token *token = &tokens->at(i);
336 printf("%s ", token_name(token));
337 fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stdout);
338 printf("\n");
431 fprintf(stderr, "%s ", token_name(token));
432 if (token->start_pos >= 0) {
433 fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stderr);
434 }
435 fprintf(stderr, "\n");
339436 }
340437}
341438
src/tokenizer.hpp-8
......@@ -43,14 +43,6 @@ struct Token {
4343 int start_column;
4444};
4545
46enum TokenizeState {
47 TokenizeStateStart,
48 TokenizeStateSymbol,
49 TokenizeStateNumber,
50 TokenizeStateString,
51 TokenizeStateSawDash,
52};
53
5446ZigList<Token> *tokenize(Buf *buf);
5547
5648void print_tokens(Buf *buf, ZigList<Token> *tokens);
test/standalone.cpp+23-2
......@@ -75,6 +75,26 @@ static void add_all_test_cases(void) {
7575 exit(0);
7676 }
7777 )SOURCE", "OK\n");
78
79 add_simple_case("comments", R"SOURCE(
80 #link("c")
81 extern {
82 fn puts(s: *mut u8) -> i32;
83 fn exit(code: i32) -> unreachable;
84 }
85
86 /**
87 * multi line doc comment
88 */
89 fn another_function() -> i32 { return 0; }
90
91 /// this is a documentation comment
92 /// doc comment line 2
93 fn _start() -> unreachable {
94 puts(/* mid-line comment /* nested */ */ "OK");
95 exit(0);
96 }
97 )SOURCE", "OK\n");
7898}
7999
80100static void run_test(TestCase *test_case) {
......@@ -83,11 +103,12 @@ static void run_test(TestCase *test_case) {
83103 Buf zig_stderr = BUF_INIT;
84104 Buf zig_stdout = BUF_INIT;
85105 int return_code;
86 os_exec_process("./zig", test_case->compiler_args, &return_code, &zig_stderr, &zig_stdout);
106 static const char *zig_exe = "./zig";
107 os_exec_process(zig_exe, test_case->compiler_args, &return_code, &zig_stderr, &zig_stdout);
87108
88109 if (return_code != 0) {
89110 printf("\nCompile failed with return code %d:\n", return_code);
90 printf("zig");
111 printf("%s", zig_exe);
91112 for (int i = 0; i < test_case->compiler_args.length; i += 1) {
92113 printf(" %s", test_case->compiler_args.at(i));
93114 }