authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-01 22:21:33-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-01 22:21:33-07:00
log34f8d80eac87a5e3bdf495163d8bc0fbd80ae83f
treeedb9b9d81a0d6552a2ac7e7de0df0f5a85288535
parent5f48463bdd843a9d8dfd55cfc389637b39fc1074

tokenizing hello.zig

6 files changed, 182 insertions(+), 178 deletions(-)

README.md+29-2
......@@ -1,5 +1,32 @@
11# zig lang
22
3C upgrade.
3An experiment in writing a low-level programming language with the intent to
4replace C. Zig intends to be a small language, yet powerful enough to write
5readable, safe, optimal, and concise code to solve any computing problem.
46
5Start with C.
7## Goals
8
9 * Ability to run arbitrary code at compile time and generate code.
10 * Completely compatible with C libraries with no wrapper necessary.
11 * Creating a C library should be a primary use case. Should be easy to export
12 an auto-generated .h file.
13 * Generics such as containers.
14 * Do not depend on libc.
15 * First class error code support.
16 * Include documentation generator.
17 * Eliminate the need for make, cmake, etc.
18 * Friendly toward package maintainers.
19 * Eliminate the need for C headers (when using zig internally).
20 * Ability to declare dependencies as Git URLS with commit locking (can
21 provide a tag or sha1).
22 * Rust-style enums.
23 * Opinionated when it makes life easier.
24 - Tab character in source code is a compile error.
25 - Whitespace at the end of line is a compile error.
26 * Resilient to parsing errors to make IDE integration work well.
27 * Source code is UTF-8.
28
29## Roadmap
30
31 * Hello, world.
32 * How should the Widget use case be solved? In Genesis I'm using C++ and inheritance.
src/buffer.cpp+21
......@@ -23,3 +23,24 @@ Buf *buf_sprintf(const char *format, ...) {
2323
2424 return buf;
2525}
26
27void buf_appendf(Buf *buf, const char *format, ...) {
28 va_list ap, ap2;
29 va_start(ap, format);
30 va_copy(ap2, ap);
31
32 int len1 = vsnprintf(nullptr, 0, format, ap);
33 assert(len1 >= 0);
34
35 size_t required_size = len1 + 1;
36
37 int orig_len = buf_len(buf);
38
39 buf_resize(buf, orig_len + required_size);
40
41 int len2 = vsnprintf(buf_ptr(buf) + orig_len, required_size, format, ap2);
42 assert(len2 == len1);
43
44 va_end(ap2);
45 va_end(ap);
46}
src/buffer.hpp+3
......@@ -93,6 +93,9 @@ static inline void buf_append_char(Buf *buf, uint8_t c) {
9393 buf_append_mem(buf, (const char *)&c, 1);
9494}
9595
96void buf_appendf(Buf *buf, const char *format, ...)
97 __attribute__ ((format (printf, 2, 3)));
98
9699static inline bool buf_eql_mem(Buf *buf, const char *mem, int mem_len) {
97100 if (buf_len(buf) != mem_len)
98101 return false;
src/main.cpp+123-170
......@@ -129,7 +129,6 @@ static Buf *fetch_file(FILE *f) {
129129
130130
131131enum TokenId {
132 TokenIdDirective,
133132 TokenIdSymbol,
134133 TokenIdLParen,
135134 TokenIdRParen,
......@@ -141,6 +140,9 @@ enum TokenId {
141140 TokenIdSemicolon,
142141 TokenIdNumberLiteral,
143142 TokenIdPlus,
143 TokenIdColon,
144 TokenIdArrow,
145 TokenIdDash,
144146};
145147
146148struct Token {
......@@ -153,14 +155,10 @@ struct Token {
153155
154156enum TokenizeState {
155157 TokenizeStateStart,
156 TokenizeStateDirective,
157 TokenizeStateDirectiveName,
158 TokenizeStateIncludeQuote,
159 TokenizeStateDirectiveEnd,
160 TokenizeStateInclude,
161158 TokenizeStateSymbol,
162 TokenizeStateString,
163159 TokenizeStateNumber,
160 TokenizeStateString,
161 TokenizeStateSawDash,
164162};
165163
166164struct Tokenize {
......@@ -171,11 +169,7 @@ struct Tokenize {
171169 int line;
172170 int column;
173171 Token *cur_tok;
174 Buf *directive_name;
175172 Buf *cur_dir_path;
176 uint8_t unquote_char;
177 int quote_start_pos;
178 Buf *include_path;
179173 ZigList<char *> *include_paths;
180174};
181175
......@@ -217,68 +211,6 @@ static void end_token(Tokenize *t) {
217211 t->cur_tok = nullptr;
218212}
219213
220static void put_back(Tokenize *t, int count) {
221 t->pos -= count;
222}
223
224static void begin_directive(Tokenize *t) {
225 t->state = TokenizeStateDirective;
226 begin_token(t, TokenIdDirective);
227 assert(!t->directive_name);
228 t->directive_name = buf_alloc();
229}
230
231static bool find_and_include_file(Tokenize *t, char *dir_path, char *file_path) {
232 Buf *full_path = buf_sprintf("%s/%s", dir_path, file_path);
233
234 FILE *f = fopen(buf_ptr(full_path), "rb");
235 if (!f)
236 return false;
237
238 Buf *contents = fetch_file(f);
239
240 buf_splice_buf(t->buf, t->pos, t->pos, contents);
241
242 return true;
243}
244
245static void render_include(Tokenize *t, Buf *target_path, char unquote_char) {
246 if (unquote_char == '"') {
247 if (find_and_include_file(t, buf_ptr(t->cur_dir_path), buf_ptr(target_path)))
248 return;
249 }
250 for (int i = 0; i < t->include_paths->length; i += 1) {
251 char *include_path = t->include_paths->at(i);
252 if (find_and_include_file(t, include_path, buf_ptr(target_path)))
253 return;
254 }
255 tokenize_error(t, "include path \"%s\" not found", buf_ptr(target_path));
256}
257
258static void end_directive(Tokenize *t) {
259 end_token(t);
260 if (t->include_path) {
261 render_include(t, t->include_path, t->unquote_char);
262 t->include_path = nullptr;
263 }
264 t->state = TokenizeStateStart;
265}
266
267static void end_directive_name(Tokenize *t) {
268 if (buf_eql_str(t->directive_name, "include")) {
269 t->state = TokenizeStateInclude;
270 t->directive_name = nullptr;
271 } else {
272 tokenize_error(t, "invalid directive name: \"%s\"", buf_ptr(t->directive_name));
273 }
274}
275
276static void end_symbol(Tokenize *t) {
277 put_back(t, 1);
278 end_token(t);
279 t->state = TokenizeStateStart;
280}
281
282214static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *cur_dir_path) {
283215 Tokenize t = {0};
284216 t.tokens = allocate<ZigList<Token>>(1);
......@@ -300,9 +232,6 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
300232 t.state = TokenizeStateNumber;
301233 begin_token(&t, TokenIdNumberLiteral);
302234 break;
303 case '#':
304 begin_directive(&t);
305 break;
306235 case '"':
307236 begin_token(&t, TokenIdStringLiteral);
308237 t.state = TokenizeStateString;
......@@ -335,79 +264,20 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
335264 begin_token(&t, TokenIdSemicolon);
336265 end_token(&t);
337266 break;
267 case ':':
268 begin_token(&t, TokenIdColon);
269 end_token(&t);
270 break;
338271 case '+':
339272 begin_token(&t, TokenIdPlus);
340273 end_token(&t);
341274 break;
342 default:
343 tokenize_error(&t, "invalid character: '%c'", c);
344 }
345 break;
346 case TokenizeStateDirective:
347 switch (c) {
348 case '\n':
349 end_directive_name(&t);
350 end_directive(&t);
351 break;
352 case ' ':
353 case '\t':
354 case '\f':
355 case '\r':
356 case 0xb:
357 break;
358 case SYMBOL_CHAR:
359 t.state = TokenizeStateDirectiveName;
360 buf_append_char(t.directive_name, c);
361 break;
362 default:
363 tokenize_error(&t, "invalid directive character: '%c'", c);
364 break;
365 }
366 break;
367 case TokenizeStateDirectiveName:
368 switch (c) {
369 case WHITESPACE:
370 end_directive_name(&t);
371 break;
372 case SYMBOL_CHAR:
373 buf_append_char(t.directive_name, c);
275 case '-':
276 begin_token(&t, TokenIdDash);
277 t.state = TokenizeStateSawDash;
374278 break;
375279 default:
376 tokenize_error(&t, "invalid directive name character: '%c'", c);
377 break;
378 }
379 break;
380 case TokenizeStateInclude:
381 switch (c) {
382 case WHITESPACE:
383 break;
384 case '<':
385 case '"':
386 t.state = TokenizeStateIncludeQuote;
387 t.quote_start_pos = t.pos;
388 t.unquote_char = (c == '<') ? '>' : '"';
389 break;
390 }
391 break;
392 case TokenizeStateIncludeQuote:
393 if (c == t.unquote_char) {
394 t.include_path = buf_slice(t.buf, t.quote_start_pos + 1, t.pos);
395 t.state = TokenizeStateDirectiveEnd;
396 }
397 break;
398 case TokenizeStateDirectiveEnd:
399 switch (c) {
400 case '\n':
401 end_directive(&t);
402 break;
403 case ' ':
404 case '\t':
405 case '\f':
406 case '\r':
407 case 0xb:
408 break;
409 default:
410 tokenize_error(&t, "expected whitespace or newline: '%c'", c);
280 tokenize_error(&t, "invalid character: '%c'", c);
411281 }
412282 break;
413283 case TokenizeStateSymbol:
......@@ -415,8 +285,10 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
415285 case SYMBOL_CHAR:
416286 break;
417287 default:
418 end_symbol(&t);
419 break;
288 t.pos -= 1;
289 end_token(&t);
290 t.state = TokenizeStateStart;
291 continue;
420292 }
421293 break;
422294 case TokenizeStateString:
......@@ -434,7 +306,22 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
434306 case DIGIT:
435307 break;
436308 default:
437 end_symbol(&t);
309 t.pos -= 1;
310 end_token(&t);
311 t.state = TokenizeStateStart;
312 continue;
313 }
314 break;
315 case TokenizeStateSawDash:
316 switch (c) {
317 case '>':
318 t.cur_tok->id = TokenIdArrow;
319 end_token(&t);
320 t.state = TokenizeStateStart;
321 break;
322 default:
323 end_token(&t);
324 t.state = TokenizeStateStart;
438325 break;
439326 }
440327 break;
......@@ -450,30 +337,17 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
450337 switch (t.state) {
451338 case TokenizeStateStart:
452339 break;
453 case TokenizeStateDirective:
454 end_directive(&t);
455 break;
456 case TokenizeStateDirectiveName:
457 end_directive_name(&t);
458 end_directive(&t);
459 break;
460 case TokenizeStateInclude:
461 tokenize_error(&t, "missing include path");
462 break;
463340 case TokenizeStateSymbol:
464 end_symbol(&t);
341 end_token(&t);
465342 break;
466343 case TokenizeStateString:
467344 tokenize_error(&t, "unterminated string");
468345 break;
469346 case TokenizeStateNumber:
470 end_symbol(&t);
347 end_token(&t);
471348 break;
472 case TokenizeStateIncludeQuote:
473 tokenize_error(&t, "unterminated include path");
474 break;
475 case TokenizeStateDirectiveEnd:
476 end_directive(&t);
349 case TokenizeStateSawDash:
350 end_token(&t);
477351 break;
478352 }
479353 assert(!t.cur_tok);
......@@ -482,7 +356,6 @@ static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *c
482356
483357static const char * token_name(Token *token) {
484358 switch (token->id) {
485 case TokenIdDirective: return "Directive";
486359 case TokenIdSymbol: return "Symbol";
487360 case TokenIdLParen: return "LParen";
488361 case TokenIdRParen: return "RParen";
......@@ -494,6 +367,9 @@ static const char * token_name(Token *token) {
494367 case TokenIdSemicolon: return "Semicolon";
495368 case TokenIdNumberLiteral: return "NumberLiteral";
496369 case TokenIdPlus: return "Plus";
370 case TokenIdColon: return "Colon";
371 case TokenIdArrow: return "Arrow";
372 case TokenIdDash: return "Dash";
497373 }
498374 return "(invalid token)";
499375}
......@@ -507,6 +383,83 @@ static void print_tokens(Buf *buf, ZigList<Token> *tokens) {
507383 }
508384}
509385
386enum NodeType {
387 NodeTypeRoot,
388};
389
390struct AstNode {
391 enum NodeType type;
392 ZigList<AstNode *> children;
393};
394
395enum AstState {
396 AstStateStart,
397};
398
399struct BuildAst {
400 Buf *buf;
401 AstNode *root;
402 AstState state;
403 int line;
404 int column;
405};
406
407__attribute__ ((format (printf, 2, 3)))
408static void ast_error(BuildAst *b, const char *format, ...) {
409 int line = b->line + 1;
410 int column = b->column + 1;
411
412 va_list ap;
413 va_start(ap, format);
414 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
415 vfprintf(stderr, format, ap);
416 fprintf(stderr, "\n");
417 va_end(ap);
418 exit(EXIT_FAILURE);
419}
420
421static inline bool mem_eql_str(const char *mem, size_t mem_len, const char *str) {
422 size_t str_len = strlen(str);
423 if (str_len != mem_len)
424 return false;
425 return memcmp(mem, str, mem_len) == 0;
426}
427
428
429static AstNode *build_ast(Buf *buf, ZigList<Token> *tokens) {
430 BuildAst b = {0};
431 b.buf = buf;
432 b.root = allocate<AstNode>(1);
433 b.root->type = NodeTypeRoot;
434
435 for (int i = 0; i < tokens->length; i += 1) {
436 Token *token = &tokens->at(i);
437 const char *token_str = buf_ptr(buf) + token->start_pos;
438 int token_len = token->end_pos - token->start_pos;
439 b.line = token->start_line;
440 b.column = token->start_column;
441 switch (b.state) {
442 case AstStateStart:
443 if (mem_eql_str(token_str, token_len, "fn")) {
444 zig_panic("TODO fn");
445 } else {
446 Buf msg = {0};
447 buf_append_str(&msg, "unexpected symbol: '");
448 buf_append_mem(&msg, token_str, token_len);
449 buf_append_str(&msg, "'");
450 ast_error(&b, "%s", buf_ptr(&msg));
451 }
452 break;
453 }
454 }
455
456 return b.root;
457}
458
459static void print_ast(AstNode *node) {
460 zig_panic("TODO");
461}
462
510463char cur_dir[1024];
511464
512465int main(int argc, char **argv) {
......@@ -559,18 +512,18 @@ int main(int argc, char **argv) {
559512
560513 Buf *in_data = fetch_file(in_f);
561514
562 fprintf(stderr, "Original source:\n%s\n", buf_ptr(in_data));
515 fprintf(stderr, "Original source:\n");
516 fprintf(stderr, "----------------\n");
517 fprintf(stderr, "%s\n", buf_ptr(in_data));
563518
564519 ZigList<Token> *tokens = tokenize(in_data, &include_paths, cur_dir_path);
565520
566521 fprintf(stderr, "\nTokens:\n");
522 fprintf(stderr, "---------\n");
567523 print_tokens(in_data, tokens);
568524
569 /*
570 Buf *preprocessed_source = preprocess(in_data, tokens, &include_paths, cur_dir_path);
571
572 fprintf(stderr, "\nPreprocessed source:\n%s\n", buf_ptr(preprocessed_source));
573 */
525 AstNode *root = build_ast(in_data, tokens);
526 print_ast(root);
574527
575528
576529 return EXIT_SUCCESS;
test/add.zig+2-2
......@@ -1,3 +1,3 @@
1int add(int a, int b) {
2 return a + b;
1pub fn add(a: int, b: int) -> int {
2 a + b
33}
test/hello.zig+4-4
......@@ -1,6 +1,6 @@
1#include <stdio.h>
2#include "add.h"
31
4int main(int argc, char **argv) {
5 fprintf(stderr, "hello: %d", add(1, 2));
2
3fn main(argc: int, argv: *mut char) -> int {
4 puts("Hello, world!\n");
5 return 0;
66}