authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-01 22:21:33-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-01 22:21:33-07:00
log34f8d80eac87a5e3bdf495163d8bc0fbd80ae83f
treeedb9b9d81a0d6552a2ac7e7de0df0f5a85288535
parent5f48463bdd843a9d8dfd55cfc389637b39fc1074

tokenizing hello.zig

6 files changed, 182 insertions(+), 178 deletions(-)

README.md+29-2
...@@ -1,5 +1,32 @@...@@ -1,5 +1,32 @@
1# zig lang1# zig lang
22
3C upgrade.3An experiment in writing a low-level programming language with the intent to
4replace C. Zig intends to be a small language, yet powerful enough to write
5readable, safe, optimal, and concise code to solve any computing problem.
46
5Start with C.7## Goals
8
9 * Ability to run arbitrary code at compile time and generate code.
10 * Completely compatible with C libraries with no wrapper necessary.
11 * Creating a C library should be a primary use case. Should be easy to export
12 an auto-generated .h file.
13 * Generics such as containers.
14 * Do not depend on libc.
15 * First class error code support.
16 * Include documentation generator.
17 * Eliminate the need for make, cmake, etc.
18 * Friendly toward package maintainers.
19 * Eliminate the need for C headers (when using zig internally).
20 * Ability to declare dependencies as Git URLS with commit locking (can
21 provide a tag or sha1).
22 * Rust-style enums.
23 * Opinionated when it makes life easier.
24 - Tab character in source code is a compile error.
25 - Whitespace at the end of line is a compile error.
26 * Resilient to parsing errors to make IDE integration work well.
27 * Source code is UTF-8.
28
29## Roadmap
30
31 * Hello, world.
32 * How should the Widget use case be solved? In Genesis I'm using C++ and inheritance.
src/buffer.cpp+21
...@@ -23,3 +23,24 @@ Buf *buf_sprintf(const char *format, ...) {...@@ -23,3 +23,24 @@ Buf *buf_sprintf(const char *format, ...) {
2323
24 return buf;24 return buf;
25}25}
26
27void buf_appendf(Buf *buf, const char *format, ...) {
28 va_list ap, ap2;
29 va_start(ap, format);
30 va_copy(ap2, ap);
31
32 int len1 = vsnprintf(nullptr, 0, format, ap);
33 assert(len1 >= 0);
34
35 size_t required_size = len1 + 1;
36
37 int orig_len = buf_len(buf);
38
39 buf_resize(buf, orig_len + required_size);
40
41 int len2 = vsnprintf(buf_ptr(buf) + orig_len, required_size, format, ap2);
42 assert(len2 == len1);
43
44 va_end(ap2);
45 va_end(ap);
46}
src/buffer.hpp+3
...@@ -93,6 +93,9 @@ static inline void buf_append_char(Buf *buf, uint8_t c) {...@@ -93,6 +93,9 @@ static inline void buf_append_char(Buf *buf, uint8_t c) {
93 buf_append_mem(buf, (const char *)&c, 1);93 buf_append_mem(buf, (const char *)&c, 1);
94}94}
9595
96void buf_appendf(Buf *buf, const char *format, ...)
97 __attribute__ ((format (printf, 2, 3)));
98
96static inline bool buf_eql_mem(Buf *buf, const char *mem, int mem_len) {99static inline bool buf_eql_mem(Buf *buf, const char *mem, int mem_len) {
97 if (buf_len(buf) != mem_len)100 if (buf_len(buf) != mem_len)
98 return false;101 return false;
src/main.cpp+123-170
...@@ -129,7 +129,6 @@ static Buf *fetch_file(FILE *f) {...@@ -129,7 +129,6 @@ static Buf *fetch_file(FILE *f) {
129129
130130
131enum TokenId {131enum TokenId {
132 TokenIdDirective,
133 TokenIdSymbol,132 TokenIdSymbol,
134 TokenIdLParen,133 TokenIdLParen,
135 TokenIdRParen,134 TokenIdRParen,
...@@ -141,6 +140,9 @@ enum TokenId {...@@ -141,6 +140,9 @@ enum TokenId {
141 TokenIdSemicolon,140 TokenIdSemicolon,
142 TokenIdNumberLiteral,141 TokenIdNumberLiteral,
143 TokenIdPlus,142 TokenIdPlus,
143 TokenIdColon,
144 TokenIdArrow,
145 TokenIdDash,
144};146};
145147
146struct Token {148struct Token {
...@@ -153,14 +155,10 @@ struct Token {...@@ -153,14 +155,10 @@ struct Token {
153155
154enum TokenizeState {156enum TokenizeState {
155 TokenizeStateStart,157 TokenizeStateStart,
156 TokenizeStateDirective,
157 TokenizeStateDirectiveName,
158 TokenizeStateIncludeQuote,
159 TokenizeStateDirectiveEnd,
160 TokenizeStateInclude,
161 TokenizeStateSymbol,158 TokenizeStateSymbol,
162 TokenizeStateString,
163 TokenizeStateNumber,159 TokenizeStateNumber,
160 TokenizeStateString,
161 TokenizeStateSawDash,
164};162};
165163
166struct Tokenize {164struct Tokenize {
...@@ -171,11 +169,7 @@ struct Tokenize {...@@ -171,11 +169,7 @@ struct Tokenize {
171 int line;169 int line;
172 int column;170 int column;
173 Token *cur_tok;171 Token *cur_tok;
174 Buf *directive_name;
175 Buf *cur_dir_path;172 Buf *cur_dir_path;
176 uint8_t unquote_char;
177 int quote_start_pos;
178 Buf *include_path;
179 ZigList<char *> *include_paths;173 ZigList<char *> *include_paths;
180};174};
181175
...@@ -217,68 +211,6 @@ static void end_token(Tokenize *t) {...@@ -217,68 +211,6 @@ static void end_token(Tokenize *t) {
217 t->cur_tok = nullptr;211 t->cur_tok = nullptr;
218}212}
219213
220static void put_back(Tokenize *t, int count) {
221 t->pos -= count;
222}
223
224static void begin_directive(Tokenize *t) {
225 t->state = TokenizeStateDirective;
226 begin_token(t, TokenIdDirective);
227 assert(!t->directive_name);
228 t->directive_name = buf_alloc();
229}
230
231static bool find_and_include_file(Tokenize *t, char *dir_path, char *file_path) {
232 Buf *full_path = buf_sprintf("%s/%s", dir_path, file_path);
233
234 FILE *f = fopen(buf_ptr(full_path), "rb");
235 if (!f)
236 return false;
237
238 Buf *contents = fetch_file(f);
239
240 buf_splice_buf(t->buf, t->pos, t->pos, contents);
241
242 return true;
243}
244
245static void render_include(Tokenize *t, Buf *target_path, char unquote_char) {
246 if (unquote_char == '"') {
247 if (find_and_include_file(t, buf_ptr(t->cur_dir_path), buf_ptr(target_path)))
248 return;
249 }
250 for (int i = 0; i < t->include_paths->length; i += 1) {
251 char *include_path = t->include_paths->at(i);
252 if (find_and_include_file(t, include_path, buf_ptr(target_path)))
253 return;
254 }
255 tokenize_error(t, "include path \"%s\" not found", buf_ptr(target_path));
256}
257
258static void end_directive(Tokenize *t) {
259 end_token(t);
260 if (t->include_path) {
261 render_include(t, t->include_path, t->unquote_char);
262 t->include_path = nullptr;
263 }
264 t->state = TokenizeStateStart;
265}
266
267static void end_directive_name(Tokenize *t) {
268 if (buf_eql_str(t->directive_name, "include")) {
269 t->state = TokenizeStateInclude;
270 t->directive_name = nullptr;
271 } else {
272 tokenize_error(t, "invalid directive name: \"%s\"", buf_ptr(t->directive_name));
273 }
274}
275
276static void end_symbol(Tokenize *t) {
277 put_back(t, 1);
278 end_token(t);
279 t->state = TokenizeStateStart;
280}
281
282static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *cur_dir_path) {214static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *cur_dir_path) {
283 Tokenize t = {0};215 Tokenize t = {0};
284 t.tokens = allocate<ZigList<Token>>(1);216 t.tokens = allocate<ZigList<Token>>(1);
...@@ -300,9 +232,6 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c...@@ -300,9 +232,6 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
300 t.state = TokenizeStateNumber;232 t.state = TokenizeStateNumber;
301 begin_token(&t, TokenIdNumberLiteral);233 begin_token(&t, TokenIdNumberLiteral);
302 break;234 break;
303 case '#':
304 begin_directive(&t);
305 break;
306 case '"':235 case '"':
307 begin_token(&t, TokenIdStringLiteral);236 begin_token(&t, TokenIdStringLiteral);
308 t.state = TokenizeStateString;237 t.state = TokenizeStateString;
...@@ -335,79 +264,20 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c...@@ -335,79 +264,20 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
335 begin_token(&t, TokenIdSemicolon);264 begin_token(&t, TokenIdSemicolon);
336 end_token(&t);265 end_token(&t);
337 break;266 break;
267 case ':':
268 begin_token(&t, TokenIdColon);
269 end_token(&t);
270 break;
338 case '+':271 case '+':
339 begin_token(&t, TokenIdPlus);272 begin_token(&t, TokenIdPlus);
340 end_token(&t);273 end_token(&t);
341 break;274 break;
342 default:275 case '-':
343 tokenize_error(&t, "invalid character: '%c'", c);276 begin_token(&t, TokenIdDash);
344 }277 t.state = TokenizeStateSawDash;
345 break;
346 case TokenizeStateDirective:
347 switch (c) {
348 case '\n':
349 end_directive_name(&t);
350 end_directive(&t);
351 break;
352 case ' ':
353 case '\t':
354 case '\f':
355 case '\r':
356 case 0xb:
357 break;
358 case SYMBOL_CHAR:
359 t.state = TokenizeStateDirectiveName;
360 buf_append_char(t.directive_name, c);
361 break;
362 default:
363 tokenize_error(&t, "invalid directive character: '%c'", c);
364 break;
365 }
366 break;
367 case TokenizeStateDirectiveName:
368 switch (c) {
369 case WHITESPACE:
370 end_directive_name(&t);
371 break;
372 case SYMBOL_CHAR:
373 buf_append_char(t.directive_name, c);
374 break;278 break;
375 default:279 default:
376 tokenize_error(&t, "invalid directive name character: '%c'", c);280 tokenize_error(&t, "invalid character: '%c'", c);
377 break;
378 }
379 break;
380 case TokenizeStateInclude:
381 switch (c) {
382 case WHITESPACE:
383 break;
384 case '<':
385 case '"':
386 t.state = TokenizeStateIncludeQuote;
387 t.quote_start_pos = t.pos;
388 t.unquote_char = (c == '<') ? '>' : '"';
389 break;
390 }
391 break;
392 case TokenizeStateIncludeQuote:
393 if (c == t.unquote_char) {
394 t.include_path = buf_slice(t.buf, t.quote_start_pos + 1, t.pos);
395 t.state = TokenizeStateDirectiveEnd;
396 }
397 break;
398 case TokenizeStateDirectiveEnd:
399 switch (c) {
400 case '\n':
401 end_directive(&t);
402 break;
403 case ' ':
404 case '\t':
405 case '\f':
406 case '\r':
407 case 0xb:
408 break;
409 default:
410 tokenize_error(&t, "expected whitespace or newline: '%c'", c);
411 }281 }
412 break;282 break;
413 case TokenizeStateSymbol:283 case TokenizeStateSymbol:
...@@ -415,8 +285,10 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c...@@ -415,8 +285,10 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
415 case SYMBOL_CHAR:285 case SYMBOL_CHAR:
416 break;286 break;
417 default:287 default:
418 end_symbol(&t);288 t.pos -= 1;
419 break;289 end_token(&t);
290 t.state = TokenizeStateStart;
291 continue;
420 }292 }
421 break;293 break;
422 case TokenizeStateString:294 case TokenizeStateString:
...@@ -434,7 +306,22 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c...@@ -434,7 +306,22 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
434 case DIGIT:306 case DIGIT:
435 break;307 break;
436 default:308 default:
437 end_symbol(&t);309 t.pos -= 1;
310 end_token(&t);
311 t.state = TokenizeStateStart;
312 continue;
313 }
314 break;
315 case TokenizeStateSawDash:
316 switch (c) {
317 case '>':
318 t.cur_tok->id = TokenIdArrow;
319 end_token(&t);
320 t.state = TokenizeStateStart;
321 break;
322 default:
323 end_token(&t);
324 t.state = TokenizeStateStart;
438 break;325 break;
439 }326 }
440 break;327 break;
...@@ -450,30 +337,17 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c...@@ -450,30 +337,17 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
450 switch (t.state) {337 switch (t.state) {
451 case TokenizeStateStart:338 case TokenizeStateStart:
452 break;339 break;
453 case TokenizeStateDirective:
454 end_directive(&t);
455 break;
456 case TokenizeStateDirectiveName:
457 end_directive_name(&t);
458 end_directive(&t);
459 break;
460 case TokenizeStateInclude:
461 tokenize_error(&t, "missing include path");
462 break;
463 case TokenizeStateSymbol:340 case TokenizeStateSymbol:
464 end_symbol(&t);341 end_token(&t);
465 break;342 break;
466 case TokenizeStateString:343 case TokenizeStateString:
467 tokenize_error(&t, "unterminated string");344 tokenize_error(&t, "unterminated string");
468 break;345 break;
469 case TokenizeStateNumber:346 case TokenizeStateNumber:
470 end_symbol(&t);347 end_token(&t);
471 break;348 break;
472 case TokenizeStateIncludeQuote:349 case TokenizeStateSawDash:
473 tokenize_error(&t, "unterminated include path");350 end_token(&t);
474 break;
475 case TokenizeStateDirectiveEnd:
476 end_directive(&t);
477 break;351 break;
478 }352 }
479 assert(!t.cur_tok);353 assert(!t.cur_tok);
...@@ -482,7 +356,6 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c...@@ -482,7 +356,6 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
482356
483static const char * token_name(Token *token) {357static const char * token_name(Token *token) {
484 switch (token->id) {358 switch (token->id) {
485 case TokenIdDirective: return "Directive";
486 case TokenIdSymbol: return "Symbol";359 case TokenIdSymbol: return "Symbol";
487 case TokenIdLParen: return "LParen";360 case TokenIdLParen: return "LParen";
488 case TokenIdRParen: return "RParen";361 case TokenIdRParen: return "RParen";
...@@ -494,6 +367,9 @@ static const char * token_name(Token *token) {...@@ -494,6 +367,9 @@ static const char * token_name(Token *token) {
494 case TokenIdSemicolon: return "Semicolon";367 case TokenIdSemicolon: return "Semicolon";
495 case TokenIdNumberLiteral: return "NumberLiteral";368 case TokenIdNumberLiteral: return "NumberLiteral";
496 case TokenIdPlus: return "Plus";369 case TokenIdPlus: return "Plus";
370 case TokenIdColon: return "Colon";
371 case TokenIdArrow: return "Arrow";
372 case TokenIdDash: return "Dash";
497 }373 }
498 return "(invalid token)";374 return "(invalid token)";
499}375}
...@@ -507,6 +383,83 @@ static void print_tokens(Buf *buf, ZigList<Token> *tokens) {...@@ -507,6 +383,83 @@ static void print_tokens(Buf *buf, ZigList<Token> *tokens) {
507 }383 }
508}384}
509385
386enum NodeType {
387 NodeTypeRoot,
388};
389
390struct AstNode {
391 enum NodeType type;
392 ZigList<AstNode *> children;
393};
394
395enum AstState {
396 AstStateStart,
397};
398
399struct BuildAst {
400 Buf *buf;
401 AstNode *root;
402 AstState state;
403 int line;
404 int column;
405};
406
407__attribute__ ((format (printf, 2, 3)))
408static void ast_error(BuildAst *b, const char *format, ...) {
409 int line = b->line + 1;
410 int column = b->column + 1;
411
412 va_list ap;
413 va_start(ap, format);
414 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
415 vfprintf(stderr, format, ap);
416 fprintf(stderr, "\n");
417 va_end(ap);
418 exit(EXIT_FAILURE);
419}
420
421static inline bool mem_eql_str(const char *mem, size_t mem_len, const char *str) {
422 size_t str_len = strlen(str);
423 if (str_len != mem_len)
424 return false;
425 return memcmp(mem, str, mem_len) == 0;
426}
427
428
429static AstNode *build_ast(Buf *buf, ZigList<Token> *tokens) {
430 BuildAst b = {0};
431 b.buf = buf;
432 b.root = allocate<AstNode>(1);
433 b.root->type = NodeTypeRoot;
434
435 for (int i = 0; i < tokens->length; i += 1) {
436 Token *token = &tokens->at(i);
437 const char *token_str = buf_ptr(buf) + token->start_pos;
438 int token_len = token->end_pos - token->start_pos;
439 b.line = token->start_line;
440 b.column = token->start_column;
441 switch (b.state) {
442 case AstStateStart:
443 if (mem_eql_str(token_str, token_len, "fn")) {
444 zig_panic("TODO fn");
445 } else {
446 Buf msg = {0};
447 buf_append_str(&msg, "unexpected symbol: '");
448 buf_append_mem(&msg, token_str, token_len);
449 buf_append_str(&msg, "'");
450 ast_error(&b, "%s", buf_ptr(&msg));
451 }
452 break;
453 }
454 }
455
456 return b.root;
457}
458
459static void print_ast(AstNode *node) {
460 zig_panic("TODO");
461}
462
510char cur_dir[1024];463char cur_dir[1024];
511464
512int main(int argc, char **argv) {465int main(int argc, char **argv) {
...@@ -559,18 +512,18 @@ int main(int argc, char **argv) {...@@ -559,18 +512,18 @@ int main(int argc, char **argv) {
559512
560 Buf *in_data = fetch_file(in_f);513 Buf *in_data = fetch_file(in_f);
561514
562 fprintf(stderr, "Original source:\n%s\n", buf_ptr(in_data));515 fprintf(stderr, "Original source:\n");
516 fprintf(stderr, "----------------\n");
517 fprintf(stderr, "%s\n", buf_ptr(in_data));
563518
564 ZigList<Token> *tokens = tokenize(in_data, &include_paths, cur_dir_path);519 ZigList<Token> *tokens = tokenize(in_data, &include_paths, cur_dir_path);
565520
566 fprintf(stderr, "\nTokens:\n");521 fprintf(stderr, "\nTokens:\n");
522 fprintf(stderr, "---------\n");
567 print_tokens(in_data, tokens);523 print_tokens(in_data, tokens);
568524
569 /*525 AstNode *root = build_ast(in_data, tokens);
570 Buf *preprocessed_source = preprocess(in_data, tokens, &include_paths, cur_dir_path);526 print_ast(root);
571
572 fprintf(stderr, "\nPreprocessed source:\n%s\n", buf_ptr(preprocessed_source));
573 */
574527
575528
576 return EXIT_SUCCESS;529 return EXIT_SUCCESS;
test/add.zig+2-2
...@@ -1,3 +1,3 @@...@@ -1,3 +1,3 @@
1int add(int a, int b) {1pub fn add(a: int, b: int) -> int {
2 return a + b;2 a + b
3}3}
test/hello.zig+4-4
...@@ -1,6 +1,6 @@...@@ -1,6 +1,6 @@
1#include <stdio.h>
2#include "add.h"
31
4int main(int argc, char **argv) {2
5 fprintf(stderr, "hello: %d", add(1, 2));3fn main(argc: int, argv: *mut char) -> int {
4 puts("Hello, world!\n");
5 return 0;
6}6}