authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-03 22:31:27-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-03 22:39:35-07:00
log7cfceeca2d5965baf6dcd45f07ebeaedbe48856f
treef0622d35db9fb0208343016230af0a7889f66391
parent303823b6b8b250580d644bacce5285efda76b731

parser generator beginnings

10 files changed, 972 insertions(+), 755 deletions(-)

CMakeLists.txt+24
......@@ -20,10 +20,21 @@ include_directories(
2020 ${CMAKE_BINARY_DIR}
2121)
2222
23set(GRAMMAR_TXT "${CMAKE_BINARY_DIR}/simple.txt")
24set(PARSER_CPP "${CMAKE_BINARY_DIR}/parser.cpp")
25
2326set(ZIG_SOURCES
2427 "${CMAKE_SOURCE_DIR}/src/main.cpp"
2528 "${CMAKE_SOURCE_DIR}/src/util.cpp"
2629 "${CMAKE_SOURCE_DIR}/src/buffer.cpp"
30 "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp"
31 ${PARSER_CPP}
32)
33
34set(PARSERGEN_SOURCES
35 "${CMAKE_SOURCE_DIR}/src/parsergen.cpp"
36 "${CMAKE_SOURCE_DIR}/src/util.cpp"
37 "${CMAKE_SOURCE_DIR}/src/buffer.cpp"
2738)
2839
2940set(CONFIGURE_OUT_FILE "${CMAKE_BINARY_DIR}/config.h")
......@@ -49,3 +60,16 @@ target_link_libraries(zig LINK_PUBLIC
4960 ${LLVM_LIBRARIES}
5061)
5162install(TARGETS zig DESTINATION bin)
63
64add_executable(parsergen ${PARSERGEN_SOURCES})
65set_target_properties(parsergen PROPERTIES
66 LINKER_LANGUAGE C
67 COMPILE_FLAGS ${EXE_CFLAGS})
68
69
70add_custom_command(
71 OUTPUT ${PARSER_CPP}
72 COMMAND parsergen ARGS ${GRAMMAR_TXT} ${PARSER_CPP}
73 DEPENDS ${GRAMMAR_TXT} ${PARSERGEN_SOURCES}
74 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
75)
README.md+18-1
......@@ -11,7 +11,7 @@ readable, safe, optimal, and concise code to solve any computing problem.
1111 * Creating a C library should be a primary use case. Should be easy to export
1212 an auto-generated .h file.
1313 * Generics such as containers.
14 * Do not depend on libc.
14 * Do not depend on libc unless explicitly imported.
1515 * First class error code support.
1616 * Include documentation generator.
1717 * Eliminate the need for make, cmake, etc.
......@@ -25,6 +25,7 @@ readable, safe, optimal, and concise code to solve any computing problem.
2525 - Whitespace at the end of line is a compile error.
2626 * Resilient to parsing errors to make IDE integration work well.
2727 * Source code is UTF-8.
28 * Shebang line OK so language can be used for "scripting" as well.
2829
2930## Roadmap
3031
......@@ -35,3 +36,19 @@ readable, safe, optimal, and concise code to solve any computing problem.
3536 * Unit tests.
3637 * Simple .so library
3738 * How should the Widget use case be solved? In Genesis I'm using C++ and inheritance.
39
40## Grammar
41
42```
43Root : FnDecl*
44FnDecl : TokenFn TokenSymbol TokenLParen list(ParamDecl, TokenComma, 0) TokenRParen (TokenArrow Type)? Block
45ParamDecl : TokenSymbol TokenColon Type
46Type : TokenSymbol | PointerType
47PointerType : TokenStar (TokenConst | TokenMut) Type
48Block : TokenLBrace Statement* Expression? TokenRBrace
49Statement : ExpressionStatement | ReturnStatement
50ExpressionStatement : Expression TokenSemicolon
51ReturnStatement : TokenReturn Expression TokenSemicolon
52Expression : TokenNumber | TokenString | FnCall
53FnCall : TokenSymbol TokenLParen list(Expression, TokenComma, 0) TokenRParen
54```
src/buffer.hpp+4-4
......@@ -49,19 +49,19 @@ static inline void buf_deinit(Buf *buf) {
4949 buf->list.deinit();
5050}
5151
52static inline void buf_init_from_mem(Buf *buf, char *ptr, int len) {
52static inline void buf_init_from_mem(Buf *buf, const char *ptr, int len) {
5353 buf->list.resize(len + 1);
5454 memcpy(buf_ptr(buf), ptr, len);
5555 buf->list.at(buf_len(buf)) = 0;
5656}
5757
58static inline Buf *buf_create_from_mem(char *ptr, int len) {
58static inline Buf *buf_create_from_mem(const char *ptr, int len) {
5959 Buf *buf = allocate<Buf>(1);
6060 buf_init_from_mem(buf, ptr, len);
6161 return buf;
6262}
6363
64static inline Buf *buf_create_from_str(char *str) {
64static inline Buf *buf_create_from_str(const char *str) {
6565 return buf_create_from_mem(str, strlen(str));
6666}
6767
......@@ -138,7 +138,7 @@ static inline Buf *buf_dirname(Buf *buf) {
138138 return buf_slice(buf, 0, i);
139139 }
140140 }
141 zig_panic("TODO buf_dirname no slash");
141 return buf_create_from_mem("", 0);
142142}
143143
144144
src/grammar.txt created+53
......@@ -0,0 +1,53 @@
1Root : many(FnDecl) token(EOF) {
2 $$ = ast_create_root($1);
3}
4
5FnDecl : token(Fn) token(Symbol) token(LParen) list(ParamDecl, token(Comma)) token(RParen) option(token(Arrow) Type) Block {
6 $$ = ast_create_fn_decl($2, $4, $6, $7);
7}
8
9ParamDecl : token(Symbol) token(Colon) Type {
10 $$ = ast_create_param_decl($1, $2);
11}
12
13Type : token(Symbol) {
14 $$ = ast_create_symbol_type($1);
15} | PointerType {
16 $$ = $1;
17}
18
19PointerType : token(Star) token(Const) Type {
20 $$ = ast_create_pointer_type($2, $3);
21} | token(Star) token(Mut) Type {
22 $$ = ast_create_pointer_type($2, $3);
23}
24
25Block : token(LBrace) many(Statement) option(Expression) token(RBrace) {
26 $$ = ast_create_block($2, $3);
27}
28
29Statement : ExpressionStatement {
30 $$ = $1;
31} | ReturnStatement {
32 $$ = $1;
33}
34
35ExpressionStatement : Expression token(Semicolon) {
36 $$ = ast_create_expression_statement($1);
37}
38
39ReturnStatement : token(Return) Expression token(Semicolon) {
40 $$ = ast_create_return_statement($2);
41}
42
43Expression : token(Number) {
44 $$ = ast_create_number($1);
45} | token(String) {
46 $$ = ast_create_string($1);
47} | FnCall {
48 $$ = $1;
49}
50
51FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) {
52 $$ = ast_create_fn_call($1, $3);
53}
src/main.cpp+37-750
......@@ -9,6 +9,8 @@
99#include "util.hpp"
1010#include "list.hpp"
1111#include "buffer.hpp"
12#include "parser.hpp"
13#include "tokenizer.hpp"
1214
1315#include <stdio.h>
1416#include <string.h>
......@@ -48,466 +50,9 @@ static Buf *fetch_file(FILE *f) {
4850 return buf;
4951}
5052
51static inline bool mem_eql_str(const char *mem, size_t mem_len, const char *str) {
52 size_t str_len = strlen(str);
53 if (str_len != mem_len)
54 return false;
55 return memcmp(mem, str, mem_len) == 0;
56}
57
58
59#define WHITESPACE \
60 ' ': \
61 case '\t': \
62 case '\n': \
63 case '\f': \
64 case '\r': \
65 case 0xb
66
67#define DIGIT \
68 '0': \
69 case '1': \
70 case '2': \
71 case '3': \
72 case '4': \
73 case '5': \
74 case '6': \
75 case '7': \
76 case '8': \
77 case '9'
78
79#define ALPHA \
80 'a': \
81 case 'b': \
82 case 'c': \
83 case 'd': \
84 case 'e': \
85 case 'f': \
86 case 'g': \
87 case 'h': \
88 case 'i': \
89 case 'j': \
90 case 'k': \
91 case 'l': \
92 case 'm': \
93 case 'n': \
94 case 'o': \
95 case 'p': \
96 case 'q': \
97 case 'r': \
98 case 's': \
99 case 't': \
100 case 'u': \
101 case 'v': \
102 case 'w': \
103 case 'x': \
104 case 'y': \
105 case 'z': \
106 case 'A': \
107 case 'B': \
108 case 'C': \
109 case 'D': \
110 case 'E': \
111 case 'F': \
112 case 'G': \
113 case 'H': \
114 case 'I': \
115 case 'J': \
116 case 'K': \
117 case 'L': \
118 case 'M': \
119 case 'N': \
120 case 'O': \
121 case 'P': \
122 case 'Q': \
123 case 'R': \
124 case 'S': \
125 case 'T': \
126 case 'U': \
127 case 'V': \
128 case 'W': \
129 case 'X': \
130 case 'Y': \
131 case 'Z'
132
133#define SYMBOL_CHAR \
134 ALPHA: \
135 case DIGIT: \
136 case '_'
137
138
139enum TokenId {
140 TokenIdSymbol,
141 TokenIdKeywordFn,
142 TokenIdKeywordReturn,
143 TokenIdKeywordMut,
144 TokenIdKeywordConst,
145 TokenIdLParen,
146 TokenIdRParen,
147 TokenIdComma,
148 TokenIdStar,
149 TokenIdLBrace,
150 TokenIdRBrace,
151 TokenIdStringLiteral,
152 TokenIdSemicolon,
153 TokenIdNumberLiteral,
154 TokenIdPlus,
155 TokenIdColon,
156 TokenIdArrow,
157 TokenIdDash,
158};
159
160struct Token {
161 TokenId id;
162 int start_pos;
163 int end_pos;
164 int start_line;
165 int start_column;
166};
167
168enum TokenizeState {
169 TokenizeStateStart,
170 TokenizeStateSymbol,
171 TokenizeStateNumber,
172 TokenizeStateString,
173 TokenizeStateSawDash,
174};
175
176struct Tokenize {
177 Buf *buf;
178 int pos;
179 TokenizeState state;
180 ZigList<Token> *tokens;
181 int line;
182 int column;
183 Token *cur_tok;
184 Buf *cur_dir_path;
185 ZigList<char *> *include_paths;
186};
187
188__attribute__ ((format (printf, 2, 3)))
189static void tokenize_error(Tokenize *t, const char *format, ...) {
190 int line;
191 int column;
192 if (t->cur_tok) {
193 line = t->cur_tok->start_line + 1;
194 column = t->cur_tok->start_column + 1;
195 } else {
196 line = t->line + 1;
197 column = t->column + 1;
198 }
199
200 va_list ap;
201 va_start(ap, format);
202 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
203 vfprintf(stderr, format, ap);
204 fprintf(stderr, "\n");
205 va_end(ap);
206 exit(EXIT_FAILURE);
207}
208
209static void begin_token(Tokenize *t, TokenId id) {
210 assert(!t->cur_tok);
211 t->tokens->add_one();
212 Token *token = &t->tokens->last();
213 token->start_line = t->line;
214 token->start_column = t->column;
215 token->id = id;
216 token->start_pos = t->pos;
217 t->cur_tok = token;
218}
219
220static void end_token(Tokenize *t) {
221 assert(t->cur_tok);
222 t->cur_tok->end_pos = t->pos + 1;
223
224 char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;
225 int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
226
227 if (mem_eql_str(token_mem, token_len, "fn")) {
228 t->cur_tok->id = TokenIdKeywordFn;
229 } else if (mem_eql_str(token_mem, token_len, "return")) {
230 t->cur_tok->id = TokenIdKeywordReturn;
231 } else if (mem_eql_str(token_mem, token_len, "mut")) {
232 t->cur_tok->id = TokenIdKeywordMut;
233 } else if (mem_eql_str(token_mem, token_len, "const")) {
234 t->cur_tok->id = TokenIdKeywordConst;
235 }
236
237 t->cur_tok = nullptr;
238}
239
240static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *cur_dir_path) {
241 Tokenize t = {0};
242 t.tokens = allocate<ZigList<Token>>(1);
243 t.buf = buf;
244 t.cur_dir_path = cur_dir_path;
245 t.include_paths = include_paths;
246 for (t.pos = 0; t.pos < buf_len(t.buf); t.pos += 1) {
247 uint8_t c = buf_ptr(t.buf)[t.pos];
248 switch (t.state) {
249 case TokenizeStateStart:
250 switch (c) {
251 case WHITESPACE:
252 break;
253 case ALPHA:
254 t.state = TokenizeStateSymbol;
255 begin_token(&t, TokenIdSymbol);
256 break;
257 case DIGIT:
258 t.state = TokenizeStateNumber;
259 begin_token(&t, TokenIdNumberLiteral);
260 break;
261 case '"':
262 begin_token(&t, TokenIdStringLiteral);
263 t.state = TokenizeStateString;
264 break;
265 case '(':
266 begin_token(&t, TokenIdLParen);
267 end_token(&t);
268 break;
269 case ')':
270 begin_token(&t, TokenIdRParen);
271 end_token(&t);
272 break;
273 case ',':
274 begin_token(&t, TokenIdComma);
275 end_token(&t);
276 break;
277 case '*':
278 begin_token(&t, TokenIdStar);
279 end_token(&t);
280 break;
281 case '{':
282 begin_token(&t, TokenIdLBrace);
283 end_token(&t);
284 break;
285 case '}':
286 begin_token(&t, TokenIdRBrace);
287 end_token(&t);
288 break;
289 case ';':
290 begin_token(&t, TokenIdSemicolon);
291 end_token(&t);
292 break;
293 case ':':
294 begin_token(&t, TokenIdColon);
295 end_token(&t);
296 break;
297 case '+':
298 begin_token(&t, TokenIdPlus);
299 end_token(&t);
300 break;
301 case '-':
302 begin_token(&t, TokenIdDash);
303 t.state = TokenizeStateSawDash;
304 break;
305 default:
306 tokenize_error(&t, "invalid character: '%c'", c);
307 }
308 break;
309 case TokenizeStateSymbol:
310 switch (c) {
311 case SYMBOL_CHAR:
312 break;
313 default:
314 t.pos -= 1;
315 end_token(&t);
316 t.state = TokenizeStateStart;
317 continue;
318 }
319 break;
320 case TokenizeStateString:
321 switch (c) {
322 case '"':
323 end_token(&t);
324 t.state = TokenizeStateStart;
325 break;
326 default:
327 break;
328 }
329 break;
330 case TokenizeStateNumber:
331 switch (c) {
332 case DIGIT:
333 break;
334 default:
335 t.pos -= 1;
336 end_token(&t);
337 t.state = TokenizeStateStart;
338 continue;
339 }
340 break;
341 case TokenizeStateSawDash:
342 switch (c) {
343 case '>':
344 t.cur_tok->id = TokenIdArrow;
345 end_token(&t);
346 t.state = TokenizeStateStart;
347 break;
348 default:
349 end_token(&t);
350 t.state = TokenizeStateStart;
351 break;
352 }
353 break;
354 }
355 if (c == '\n') {
356 t.line += 1;
357 t.column = 0;
358 } else {
359 t.column += 1;
360 }
361 }
362 // EOF
363 switch (t.state) {
364 case TokenizeStateStart:
365 break;
366 case TokenizeStateSymbol:
367 end_token(&t);
368 break;
369 case TokenizeStateString:
370 tokenize_error(&t, "unterminated string");
371 break;
372 case TokenizeStateNumber:
373 end_token(&t);
374 break;
375 case TokenizeStateSawDash:
376 end_token(&t);
377 break;
378 }
379 assert(!t.cur_tok);
380 return t.tokens;
381}
382
383static const char * token_name(Token *token) {
384 switch (token->id) {
385 case TokenIdSymbol: return "Symbol";
386 case TokenIdKeywordFn: return "Fn";
387 case TokenIdKeywordConst: return "Const";
388 case TokenIdKeywordMut: return "Mut";
389 case TokenIdKeywordReturn: return "Return";
390 case TokenIdLParen: return "LParen";
391 case TokenIdRParen: return "RParen";
392 case TokenIdComma: return "Comma";
393 case TokenIdStar: return "Star";
394 case TokenIdLBrace: return "LBrace";
395 case TokenIdRBrace: return "RBrace";
396 case TokenIdStringLiteral: return "StringLiteral";
397 case TokenIdSemicolon: return "Semicolon";
398 case TokenIdNumberLiteral: return "NumberLiteral";
399 case TokenIdPlus: return "Plus";
400 case TokenIdColon: return "Colon";
401 case TokenIdArrow: return "Arrow";
402 case TokenIdDash: return "Dash";
403 }
404 return "(invalid token)";
405}
406
407static void print_tokens(Buf *buf, ZigList<Token> *tokens) {
408 for (int i = 0; i < tokens->length; i += 1) {
409 Token *token = &tokens->at(i);
410 printf("%s ", token_name(token));
411 fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stdout);
412 printf("\n");
413 }
414}
415
416struct AstNode;
417
418enum NodeType {
419 NodeTypeRoot,
420 NodeTypeFnDecl,
421 NodeTypeParam,
422 NodeTypeType,
423 NodeTypeBlock,
424};
425
426struct AstNodeFnDecl {
427 Buf name;
428 ZigList<AstNode *> params;
429 AstNode *return_type;
430 AstNode *body;
431};
432
433struct AstNodeRoot {
434 ZigList<AstNode *> fn_decls;
435};
436
437enum AstNodeTypeType {
438 AstNodeTypeTypePrimitive,
439 AstNodeTypeTypePointer,
440};
441
442enum AstPrimitiveType {
443 AstPrimitiveTypeVoid,
444 AstPrimitiveTypeU8,
445 AstPrimitiveTypeI8,
446 AstPrimitiveTypeU16,
447 AstPrimitiveTypeI16,
448 AstPrimitiveTypeU32,
449 AstPrimitiveTypeI32,
450 AstPrimitiveTypeU64,
451 AstPrimitiveTypeI64,
452 AstPrimitiveTypeUSize,
453 AstPrimitiveTypeISize,
454 AstPrimitiveTypeF32,
455 AstPrimitiveTypeF64,
456};
457
458
459struct AstNodeType {
460 AstNodeTypeType type;
461 AstPrimitiveType primitive_type;
462 AstNode *pointer_type;
463 bool is_const;
464};
465
466struct AstNodeParam {
467 Buf name;
468 AstNode *type;
469};
470
471enum AstState {
472 AstStateStart,
473 AstStateFn,
474 AstStateFnLParen,
475 AstStateFnParamName,
476 AstStateParamColon,
477 AstStateType,
478 AstStateTypeEnd,
479 AstStateFnParamComma,
480 AstStateFnDeclArrow,
481 AstStateFnDeclBlock,
482 AstStatePointerType,
483 AstStateBlock,
484};
485
486struct AstNode {
487 enum NodeType type;
488 AstNode *parent;
489 AstState prev_state;
490 union {
491 AstNodeRoot root;
492 AstNodeFnDecl fn_decl;
493 AstNodeType type;
494 AstNodeParam param;
495 } data;
496};
497
498struct BuildAst {
499 Buf *buf;
500 AstNode *root;
501 AstState state;
502 int line;
503 int column;
504 AstNode *cur_node;
505};
506
507__attribute__ ((format (printf, 2, 3)))
508static void ast_error(BuildAst *b, const char *format, ...) {
509 int line = b->line + 1;
510 int column = b->column + 1;
53void ast_error(Token *token, const char *format, ...) {
54 int line = token->start_line + 1;
55 int column = token->start_column + 1;
51156
51257 va_list ap;
51358 va_start(ap, format);
......@@ -518,37 +63,35 @@ static void ast_error(BuildAst *b, const char *format, ...) {
51863 exit(EXIT_FAILURE);
51964}
52065
521static AstNode *ast_create_node(BuildAst *b, NodeType type) {
522 AstNode *child = allocate<AstNode>(1);
523 child->prev_state = b->state;
524 child->parent = b->cur_node;
525 child->type = type;
526 return child;
527}
528
529static void ast_make_node_current(BuildAst *b, AstNode *node) {
530 b->cur_node = node;
531}
532
533static void ast_up_stack(BuildAst *b) {
534 assert(b->cur_node->parent);
535 b->state = b->cur_node->prev_state;
536 b->cur_node = b->cur_node->parent;
537}
538
539
54066static const char *node_type_str(NodeType node_type) {
54167 switch (node_type) {
542 case NodeTypeRoot: return "Root";
543 case NodeTypeFnDecl: return "FnDecl";
544 case NodeTypeParam: return "Param";
545 case NodeTypeType: return "Type";
546 case NodeTypeBlock: return "Block";
68 case NodeTypeRoot:
69 return "Root";
70 case NodeTypeFnDecl:
71 return "FnDecl";
72 case NodeTypeParamDecl:
73 return "ParamDecl";
74 case NodeTypeType:
75 return "Type";
76 case NodeTypePointerType:
77 return "PointerType";
78 case NodeTypeBlock:
79 return "Block";
80 case NodeTypeStatement:
81 return "Statement";
82 case NodeTypeExpressionStatement:
83 return "ExpressionStatement";
84 case NodeTypeReturnStatement:
85 return "ReturnStatement";
86 case NodeTypeExpression:
87 return "Expression";
88 case NodeTypeFnCall:
89 return "FnCall";
54790 }
54891 zig_panic("unreachable");
54992}
55093
551static void print_ast(AstNode *node, int indent) {
94static void ast_print(AstNode *node, int indent) {
55295 for (int i = 0; i < indent; i += 1) {
55396 fprintf(stderr, " ");
55497 }
......@@ -558,7 +101,7 @@ static void print_ast(AstNode *node, int indent) {
558101 fprintf(stderr, "%s\n", node_type_str(node->type));
559102 for (int i = 0; i < node->data.root.fn_decls.length; i += 1) {
560103 AstNode *child = node->data.root.fn_decls.at(i);
561 print_ast(child, indent + 2);
104 ast_print(child, indent + 2);
562105 }
563106 break;
564107 case NodeTypeFnDecl:
......@@ -568,276 +111,19 @@ static void print_ast(AstNode *node, int indent) {
568111
569112 for (int i = 0; i < node->data.fn_decl.params.length; i += 1) {
570113 AstNode *child = node->data.fn_decl.params.at(i);
571 print_ast(child, indent + 2);
114 ast_print(child, indent + 2);
572115 }
573116
574 print_ast(node->data.fn_decl.return_type, indent + 2);
117 ast_print(node->data.fn_decl.return_type, indent + 2);
575118
576 print_ast(node->data.fn_decl.body, indent + 2);
119 ast_print(node->data.fn_decl.body, indent + 2);
577120
578121 break;
579122 }
580 case NodeTypeParam:
581 {
582 Buf *name_buf = &node->data.param.name;
583 fprintf(stderr, "%s '%s'\n", node_type_str(node->type), buf_ptr(name_buf));
584
585 print_ast(node->data.param.type, indent + 2);
586 break;
587 }
588 case NodeTypeType:
123 default:
589124 fprintf(stderr, "%s\n", node_type_str(node->type));
590125 break;
591 case NodeTypeBlock:
592 fprintf(stderr, "%s\n", node_type_str(node->type));
593 break;
594 }
595}
596
597static void ast_end_fn_param_list(BuildAst *b) {
598 b->state = AstStateFnDeclArrow;
599}
600
601static AstNode *build_ast(Buf *buf, ZigList<Token> *tokens) {
602 BuildAst b = {0};
603 b.state = AstStateStart;
604 b.buf = buf;
605 b.root = allocate<AstNode>(1);
606 b.root->type = NodeTypeRoot;
607 b.cur_node = b.root;
608
609 for (int i = 0; i < tokens->length; i += 1) {
610 Token *token = &tokens->at(i);
611 char *token_mem = buf_ptr(buf) + token->start_pos;
612 int token_len = token->end_pos - token->start_pos;
613 b.line = token->start_line;
614 b.column = token->start_column;
615 switch (b.state) {
616 case AstStateStart:
617 assert(b.cur_node->type == NodeTypeRoot);
618 if (token->id == TokenIdKeywordFn) {
619 AstNode *child = ast_create_node(&b, NodeTypeFnDecl);
620 b.cur_node->data.root.fn_decls.append(child);
621 ast_make_node_current(&b, child);
622 b.state = AstStateFn;
623 } else {
624 Buf msg = {0};
625 buf_appendf(&msg, "unexpected %s: '", token_name(token));
626 buf_append_mem(&msg, token_mem, token_len);
627 buf_append_str(&msg, "'");
628 ast_error(&b, "%s", buf_ptr(&msg));
629 break;
630 }
631 break;
632 case AstStateFn:
633 if (token->id != TokenIdSymbol)
634 ast_error(&b, "expected symbol");
635 buf_init_from_mem(&b.cur_node->data.fn_decl.name, token_mem, token_len);
636 b.state = AstStateFnLParen;
637 break;
638 case AstStateFnLParen:
639 if (token->id != TokenIdLParen)
640 ast_error(&b, "expected '('");
641 b.state = AstStateFnParamName;
642 break;
643 case AstStateFnParamName:
644 switch (token->id) {
645 case TokenIdSymbol:
646 {
647 b.state = AstStateFnParamComma;
648 AstNode *child = ast_create_node(&b, NodeTypeParam);
649 buf_init_from_mem(&child->data.param.name, token_mem, token_len);
650 b.cur_node->data.fn_decl.params.append(child);
651 ast_make_node_current(&b, child);
652 b.state = AstStateParamColon;
653 break;
654 }
655 case TokenIdRParen:
656 ast_end_fn_param_list(&b);
657 break;
658 default:
659 ast_error(&b, "expected parameter name");
660 break;
661 }
662 break;
663 case AstStateParamColon:
664 {
665 if (token->id != TokenIdColon)
666 ast_error(&b, "expected ':'");
667 assert(b.cur_node->type == NodeTypeParam);
668 b.state = AstStateTypeEnd;
669 AstNode *child = ast_create_node(&b, NodeTypeType);
670 b.cur_node->data.param.type = child;
671 ast_make_node_current(&b, child);
672 b.state = AstStateType;
673 break;
674 }
675 case AstStateType:
676 switch (token->id) {
677 case TokenIdSymbol:
678 assert(b.cur_node->type == NodeTypeType);
679 if (mem_eql_str(token_mem, token_len, "u8")) {
680 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
681 b.cur_node->data.type.primitive_type = AstPrimitiveTypeU8;
682 ast_up_stack(&b);
683 } else if (mem_eql_str(token_mem, token_len, "i8")) {
684 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
685 b.cur_node->data.type.primitive_type = AstPrimitiveTypeI8;
686 ast_up_stack(&b);
687 } else if (mem_eql_str(token_mem, token_len, "u16")) {
688 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
689 b.cur_node->data.type.primitive_type = AstPrimitiveTypeU16;
690 ast_up_stack(&b);
691 } else if (mem_eql_str(token_mem, token_len, "i16")) {
692 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
693 b.cur_node->data.type.primitive_type = AstPrimitiveTypeI16;
694 ast_up_stack(&b);
695 } else if (mem_eql_str(token_mem, token_len, "u32")) {
696 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
697 b.cur_node->data.type.primitive_type = AstPrimitiveTypeU32;
698 ast_up_stack(&b);
699 } else if (mem_eql_str(token_mem, token_len, "i32")) {
700 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
701 b.cur_node->data.type.primitive_type = AstPrimitiveTypeI32;
702 ast_up_stack(&b);
703 } else if (mem_eql_str(token_mem, token_len, "u64")) {
704 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
705 b.cur_node->data.type.primitive_type = AstPrimitiveTypeU64;
706 ast_up_stack(&b);
707 } else if (mem_eql_str(token_mem, token_len, "i64")) {
708 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
709 b.cur_node->data.type.primitive_type = AstPrimitiveTypeI64;
710 ast_up_stack(&b);
711 } else if (mem_eql_str(token_mem, token_len, "usize")) {
712 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
713 b.cur_node->data.type.primitive_type = AstPrimitiveTypeUSize;
714 ast_up_stack(&b);
715 } else if (mem_eql_str(token_mem, token_len, "isize")) {
716 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
717 b.cur_node->data.type.primitive_type = AstPrimitiveTypeISize;
718 ast_up_stack(&b);
719 } else if (mem_eql_str(token_mem, token_len, "f32")) {
720 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
721 b.cur_node->data.type.primitive_type = AstPrimitiveTypeF32;
722 ast_up_stack(&b);
723 } else if (mem_eql_str(token_mem, token_len, "f64")) {
724 b.cur_node->data.type.type = AstNodeTypeTypePrimitive;
725 b.cur_node->data.type.primitive_type = AstPrimitiveTypeF64;
726 ast_up_stack(&b);
727 } else {
728 Buf msg = {0};
729 buf_append_str(&msg, "invalid primitive type: '");
730 buf_append_mem(&msg, token_mem, token_len);
731 buf_append_str(&msg, "'");
732 ast_error(&b, "%s", buf_ptr(&msg));
733 }
734 break;
735 case TokenIdStar:
736 b.cur_node->data.type.type = AstNodeTypeTypePointer;
737 b.state = AstStatePointerType;
738 break;
739 default:
740 ast_error(&b, "expected type name");
741 break;
742 }
743 break;
744 case AstStatePointerType:
745 {
746 if (token->id == TokenIdKeywordMut) {
747 b.cur_node->data.type.is_const = false;
748 } else if (token->id == TokenIdKeywordConst) {
749 b.cur_node->data.type.is_const = true;
750 } else {
751 ast_error(&b, "expected 'mut' or 'const'");
752 }
753 b.state = AstStateTypeEnd;
754 AstNode *child = ast_create_node(&b, NodeTypeType);
755 b.cur_node->data.type.pointer_type = child;
756 ast_make_node_current(&b, child);
757 b.state = AstStateType;
758 break;
759 }
760 case AstStateTypeEnd:
761 ast_up_stack(&b);
762 i -= 1;
763 continue;
764 case AstStateFnParamComma:
765 switch (token->id) {
766 case TokenIdComma:
767 b.state = AstStateFnParamName;
768 break;
769 case TokenIdRParen:
770 ast_end_fn_param_list(&b);
771 break;
772 default:
773 ast_error(&b, "expected ',' or ')'");
774 break;
775
776 }
777 break;
778 case AstStateFnDeclArrow:
779 switch (token->id) {
780 case TokenIdArrow:
781 {
782 assert(b.cur_node->type == NodeTypeFnDecl);
783 b.state = AstStateFnDeclBlock;
784 AstNode *child = ast_create_node(&b, NodeTypeType);
785 b.cur_node->data.fn_decl.return_type = child;
786 ast_make_node_current(&b, child);
787 b.state = AstStateType;
788 break;
789 }
790 case TokenIdLBrace:
791 {
792 AstNode *node = ast_create_node(&b, NodeTypeType);
793 node->data.type.type = AstNodeTypeTypePrimitive;
794 node->data.type.primitive_type = AstPrimitiveTypeVoid;
795 b.cur_node->data.fn_decl.return_type = node;
796
797 b.state = AstStateTypeEnd;
798 AstNode *child = ast_create_node(&b, NodeTypeBlock);
799 b.cur_node->data.fn_decl.body = child;
800 ast_make_node_current(&b, child);
801 b.state = AstStateBlock;
802 break;
803 }
804 default:
805 ast_error(&b, "expected '->' or '}'");
806 break;
807 }
808 break;
809 case AstStateFnDeclBlock:
810 {
811 if (token->id != TokenIdLBrace)
812 ast_error(&b, "expected '{'");
813
814 b.state = AstStateTypeEnd;
815 AstNode *child = ast_create_node(&b, NodeTypeBlock);
816 b.cur_node->data.fn_decl.body = child;
817 ast_make_node_current(&b, child);
818 b.state = AstStateBlock;
819 break;
820 }
821 case AstStateBlock:
822 switch (token->id) {
823 case TokenIdSymbol:
824 zig_panic("TODO symbol");
825 break;
826 default:
827 {
828 Buf msg = {0};
829 buf_appendf(&msg, "unexpected %s: '", token_name(token));
830 buf_append_mem(&msg, token_mem, token_len);
831 buf_append_str(&msg, "'");
832 ast_error(&b, "%s", buf_ptr(&msg));
833 break;
834 }
835 }
836 break;
837 }
838126 }
839
840 return b.root;
841127}
842128
843129char cur_dir[1024];
......@@ -896,14 +182,15 @@ int main(int argc, char **argv) {
896182 fprintf(stderr, "----------------\n");
897183 fprintf(stderr, "%s\n", buf_ptr(in_data));
898184
899 ZigList<Token> *tokens = tokenize(in_data, &include_paths, cur_dir_path);
185 ZigList<Token> *tokens = tokenize(in_data, cur_dir_path);
900186
901187 fprintf(stderr, "\nTokens:\n");
902188 fprintf(stderr, "---------\n");
903189 print_tokens(in_data, tokens);
904190
905 AstNode *root = build_ast(in_data, tokens);
906 print_ast(root, 0);
191 AstNode *root = ast_parse(in_data, tokens);
192 assert(root);
193 ast_print(root, 0);
907194
908195
909196 return EXIT_SUCCESS;
src/parser.hpp created+87
......@@ -0,0 +1,87 @@
1#ifndef ZIG_PARSER_HPP
2#define ZIG_PARSER_HPP
3
4#include "list.hpp"
5#include "buffer.hpp"
6#include "tokenizer.hpp"
7
8struct AstNode;
9
10enum NodeType {
11 NodeTypeRoot,
12 NodeTypeFnDecl,
13 NodeTypeParamDecl,
14 NodeTypeType,
15 NodeTypePointerType,
16 NodeTypeBlock,
17 NodeTypeStatement,
18 NodeTypeExpressionStatement,
19 NodeTypeReturnStatement,
20 NodeTypeExpression,
21 NodeTypeFnCall,
22};
23
24struct AstNodeRoot {
25 ZigList<AstNode *> fn_decls;
26};
27
28struct AstNodeFnDecl {
29 Buf name;
30 ZigList<AstNode *> params;
31 AstNode *return_type;
32 AstNode *body;
33};
34
35struct AstNodeParamDecl {
36 Buf name;
37 AstNode *type;
38};
39
40enum AstNodeTypeType {
41 AstNodeTypeTypePrimitive,
42 AstNodeTypeTypePointer,
43};
44
45struct AstNodeType {
46 AstNodeTypeType type;
47 AstNode *child;
48};
49
50struct AstNodePointerType {
51 AstNode *const_or_mut;
52 AstNode *type;
53};
54
55struct AstNodeBlock {
56 ZigList<AstNode *> expressions;
57};
58
59struct AstNodeExpression {
60 AstNode *child;
61};
62
63struct AstNodeFnCall {
64 Buf name;
65 ZigList<AstNode *> params;
66};
67
68struct AstNode {
69 enum NodeType type;
70 AstNode *parent;
71 union {
72 AstNodeRoot root;
73 AstNodeFnDecl fn_decl;
74 AstNodeType type;
75 AstNodeParamDecl param_decl;
76 AstNodeBlock block;
77 AstNodeExpression expression;
78 AstNodeFnCall fn_call;
79 } data;
80};
81
82__attribute__ ((format (printf, 2, 3)))
83void ast_error(Token *token, const char *format, ...);
84
85AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens);
86
87#endif
src/parsergen.cpp created+340
......@@ -0,0 +1,340 @@
1/*
2 * Copyright (c) 2015 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8#include "util.hpp"
9#include "buffer.hpp"
10#include "list.hpp"
11
12#include <stdio.h>
13#include <sys/types.h>
14#include <sys/stat.h>
15#include <unistd.h>
16#include <errno.h>
17#include <limits.h>
18
19static Buf *fetch_file(FILE *f) {
20 int fd = fileno(f);
21 struct stat st;
22 if (fstat(fd, &st))
23 zig_panic("unable to stat file: %s", strerror(errno));
24 off_t big_size = st.st_size;
25 if (big_size > INT_MAX)
26 zig_panic("file too big");
27 int size = (int)big_size;
28
29 Buf *buf = buf_alloc_fixed(size);
30 size_t amt_read = fread(buf_ptr(buf), 1, buf_len(buf), f);
31 if (amt_read != (size_t)buf_len(buf))
32 zig_panic("error reading: %s", strerror(errno));
33
34 return buf;
35}
36
37static int usage(const char *arg0) {
38 fprintf(stderr, "Usage: %s in-grammar.txt out-parser.c\n", arg0);
39 return 1;
40}
41
42struct Token {
43 Buf name;
44 int id;
45};
46
47struct RuleNode;
48
49struct RuleTuple {
50 ZigList<RuleNode *> children;
51};
52
53struct RuleMany {
54 RuleNode *child;
55};
56
57struct RuleOption {
58 ZigList<RuleNode *> child;
59};
60
61struct RuleOr {
62 ZigList<RuleTuple *> children;
63};
64
65struct RuleToken {
66 Token *token;
67};
68
69struct RuleBlock {
70 Buf *body;
71};
72
73struct RuleList {
74 RuleNode *rule;
75 RuleToken *separator;
76};
77
78struct RuleSubRule {
79 RuleNode *child;
80};
81
82enum RuleNodeType {
83 RuleNodeTypeTuple,
84 RuleNodeTypeMany,
85 RuleNodeTypeList,
86 RuleNodeTypeOption,
87 RuleNodeTypeOr,
88 RuleNodeTypeToken,
89 RuleNodeTypeSubRule,
90};
91
92struct RuleNode {
93 RuleNodeType type;
94 union {
95 RuleTuple tuple;
96 RuleMany many;
97 RuleList list;
98 RuleOption option;
99 RuleOr _or;
100 RuleToken token;
101 RuleSubRule sub_rule;
102 };
103};
104
105enum ParserStateType {
106 ParserStateTypeError,
107 ParserStateTypeOk,
108};
109
110struct ParserStateError {
111 Buf *msg;
112};
113
114struct ParserState {
115 ParserStateType type;
116 // One for each token ID.
117 ParserState **transition;
118 int index;
119 union {
120 ParserStateError error;
121 };
122};
123
124struct Gen {
125 ParserState *cur_state;
126 ZigList<ParserState *> transition_table;
127 ZigList<Token *> tokens;
128 RuleNode *root;
129};
130
131static ParserState *create_state(Gen *g, ParserStateType type) {
132 ParserState *state = allocate<ParserState>(1);
133 state->type = type;
134 state->index = g->transition_table.length;
135 state->transition = allocate<ParserState*>(g->tokens.length);
136 g->transition_table.append(state);
137 return state;
138}
139
140static void fill_state_with_transition(Gen *g, ParserState *source, ParserState *dest) {
141 for (int i = 0; i < g->tokens.length; i += 1) {
142 source->transition[i] = dest;
143 }
144}
145
146static void gen(Gen *g, RuleNode *node) {
147 switch (node->type) {
148 case RuleNodeTypeToken:
149 {
150 ParserState *ok_state = create_state(g, ParserStateTypeOk);
151 ParserState *err_state = create_state(g, ParserStateTypeError);
152
153 err_state->error.msg = buf_sprintf("expected token '%s'", buf_ptr(&node->token.token->name));
154
155 fill_state_with_transition(g, g->cur_state, err_state);
156 g->cur_state->transition[node->token.token->id] = ok_state;
157 g->cur_state = ok_state;
158 }
159 break;
160 case RuleNodeTypeTuple:
161 {
162 for (int i = 0; i < node->tuple.children.length; i += 1) {
163 RuleNode *child = node->tuple.children.at(i);
164 gen(g, child);
165 }
166 }
167 break;
168 case RuleNodeTypeMany:
169 zig_panic("TODO");
170 break;
171 case RuleNodeTypeList:
172 zig_panic("TODO");
173 break;
174 case RuleNodeTypeOption:
175 zig_panic("TODO");
176 break;
177 case RuleNodeTypeOr:
178 zig_panic("TODO");
179 break;
180 case RuleNodeTypeSubRule:
181 zig_panic("TODO");
182 break;
183 }
184}
185
186static Token *find_token_by_name(Gen *g, Buf *name) {
187 for (int i = 0; i < g->tokens.length; i += 1) {
188 Token *token = g->tokens.at(i);
189 if (buf_eql_buf(name, &token->name))
190 return token;
191 }
192 return nullptr;
193}
194
195static Token *find_or_create_token(Gen *g, Buf *name) {
196 Token *token = find_token_by_name(g, name);
197 if (!token) {
198 token = allocate<Token>(1);
199 token->id = g->tokens.length;
200 buf_init_from_mem(&token->name, buf_ptr(name), buf_len(name));
201 g->tokens.append(token);
202 }
203 return token;
204}
205
206int main(int argc, char **argv) {
207 const char *in_filename = argv[1];
208 const char *out_filename = argv[2];
209
210 if (!in_filename || !out_filename)
211 return usage(argv[0]);
212
213 FILE *in_f;
214 if (strcmp(in_filename, "-") == 0) {
215 in_f = stdin;
216 } else {
217 in_f = fopen(in_filename, "rb");
218 }
219
220 FILE *out_f;
221 if (strcmp(out_filename, "-") == 0) {
222 out_f = stdout;
223 } else {
224 out_f = fopen(out_filename, "wb");
225 }
226
227 if (!in_f || !out_f)
228 zig_panic("unable to open file(s)");
229
230 Buf *in_buf = fetch_file(in_f);
231
232 ZigList<RuleNode *> rules = {0};
233 Gen g = {0};
234
235 //zig_panic("TODO initialize rules");
236 {
237 Token *star_token = find_or_create_token(&g, buf_create_from_str("Star"));
238 Token *lparen_token = find_or_create_token(&g, buf_create_from_str("LParen"));
239 Token *eof_token = find_or_create_token(&g, buf_create_from_str("Eof"));
240
241 RuleNode *root = allocate<RuleNode>(1);
242 root->type = RuleNodeTypeTuple;
243
244 RuleNode *star_node = allocate<RuleNode>(1);
245 star_node->type = RuleNodeTypeToken;
246 star_node->token.token = star_token;
247 root->tuple.children.append(star_node);
248
249 RuleNode *lparen_node = allocate<RuleNode>(1);
250 lparen_node->type = RuleNodeTypeToken;
251 lparen_node->token.token = lparen_token;
252 root->tuple.children.append(lparen_node);
253
254 RuleNode *eof_node = allocate<RuleNode>(1);
255 eof_node->type = RuleNodeTypeToken;
256 eof_node->token.token = eof_token;
257 root->tuple.children.append(eof_node);
258
259 rules.append(root);
260 }
261
262 g.root = rules.at(0);
263
264 g.cur_state = create_state(&g, ParserStateTypeOk);
265 gen(&g, g.root);
266
267
268
269 (void)in_buf;
270
271 fprintf(out_f, "/* This file is auto-generated by parsergen.cpp */\n");
272 fprintf(out_f, "#include \"src/parser.hpp\"\n");
273 fprintf(out_f, "#include <stdio.h>\n");
274
275 fprintf(out_f, "\n");
276 fprintf(out_f, "/*\n");
277 fprintf(out_f, "enum TokenId {\n");
278 for (int i = 0; i < g.tokens.length; i += 1) {
279 Token *token = g.tokens.at(i);
280 fprintf(out_f, " TokenId%s = %d,\n", buf_ptr(&token->name), token->id);
281 }
282 fprintf(out_f, "};\n");
283 fprintf(out_f, "*/\n");
284 for (int i = 0; i < g.tokens.length; i += 1) {
285 Token *token = g.tokens.at(i);
286 fprintf(out_f, "static_assert(TokenId%s == %d, \"wrong token id\");\n",
287 buf_ptr(&token->name), token->id);
288 }
289
290 fprintf(out_f, "AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens) {\n");
291
292 fprintf(out_f, " static const int transition[%d][%d] = {\n", g.transition_table.length, g.tokens.length);
293 for (int state_index = 0; state_index < g.transition_table.length; state_index += 1) {
294 ParserState *state = g.transition_table.at(state_index);
295 fprintf(out_f, " {\n");
296
297 for (int token_id = 0; token_id < g.tokens.length; token_id += 1) {
298 ParserState *dest = state->transition[token_id];
299 fprintf(out_f, " %d,\n", dest ? dest->index : -1);
300 }
301
302 fprintf(out_f, " },\n");
303 }
304 fprintf(out_f, " };\n");
305
306
307 fprintf(out_f, " int state = 0;\n");
308 fprintf(out_f, " AstNode *root = nullptr;\n");
309
310 fprintf(out_f, " for (int i = 0; i < tokens->length; i += 1) {\n");
311 fprintf(out_f, " Token *token = &tokens->at(i);\n");
312 fprintf(out_f, " switch (state) {\n");
313
314 for (int i = 0; i < g.transition_table.length; i += 1) {
315 ParserState *state = g.transition_table.at(i);
316 fprintf(out_f, " case %d:\n", i);
317 fprintf(out_f, " fprintf(stderr, \"state = %%d\\n\", state);\n");
318 switch (state->type) {
319 case ParserStateTypeError:
320 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(state->error.msg));
321 break;
322 case ParserStateTypeOk:
323 fprintf(out_f, " assert(transition[%d][token->id] >= 0);\n", state->index);
324 fprintf(out_f, " assert(transition[%d][token->id] < %d);\n",
325 state->index, g.transition_table.length);
326 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
327 break;
328 }
329 fprintf(out_f, " break;\n");
330 }
331 fprintf(out_f, " default:\n");
332 fprintf(out_f, " zig_panic(\"unreachable\");\n");
333
334 fprintf(out_f, " }\n");
335 fprintf(out_f, " }\n");
336 fprintf(out_f, " return root;\n");
337 fprintf(out_f, "}\n");
338
339
340}
src/tokenizer.cpp created+328
......@@ -0,0 +1,328 @@
1#include "tokenizer.hpp"
2#include "util.hpp"
3
4#include <stdarg.h>
5#include <stdlib.h>
6#include <stdio.h>
7
8#define WHITESPACE \
9 ' ': \
10 case '\t': \
11 case '\n': \
12 case '\f': \
13 case '\r': \
14 case 0xb
15
16#define DIGIT \
17 '0': \
18 case '1': \
19 case '2': \
20 case '3': \
21 case '4': \
22 case '5': \
23 case '6': \
24 case '7': \
25 case '8': \
26 case '9'
27
28#define ALPHA \
29 'a': \
30 case 'b': \
31 case 'c': \
32 case 'd': \
33 case 'e': \
34 case 'f': \
35 case 'g': \
36 case 'h': \
37 case 'i': \
38 case 'j': \
39 case 'k': \
40 case 'l': \
41 case 'm': \
42 case 'n': \
43 case 'o': \
44 case 'p': \
45 case 'q': \
46 case 'r': \
47 case 's': \
48 case 't': \
49 case 'u': \
50 case 'v': \
51 case 'w': \
52 case 'x': \
53 case 'y': \
54 case 'z': \
55 case 'A': \
56 case 'B': \
57 case 'C': \
58 case 'D': \
59 case 'E': \
60 case 'F': \
61 case 'G': \
62 case 'H': \
63 case 'I': \
64 case 'J': \
65 case 'K': \
66 case 'L': \
67 case 'M': \
68 case 'N': \
69 case 'O': \
70 case 'P': \
71 case 'Q': \
72 case 'R': \
73 case 'S': \
74 case 'T': \
75 case 'U': \
76 case 'V': \
77 case 'W': \
78 case 'X': \
79 case 'Y': \
80 case 'Z'
81
82#define SYMBOL_CHAR \
83 ALPHA: \
84 case DIGIT: \
85 case '_'
86
87
88struct Tokenize {
89 Buf *buf;
90 int pos;
91 TokenizeState state;
92 ZigList<Token> *tokens;
93 int line;
94 int column;
95 Token *cur_tok;
96 Buf *cur_dir_path;
97};
98
99__attribute__ ((format (printf, 2, 3)))
100static void tokenize_error(Tokenize *t, const char *format, ...) {
101 int line;
102 int column;
103 if (t->cur_tok) {
104 line = t->cur_tok->start_line + 1;
105 column = t->cur_tok->start_column + 1;
106 } else {
107 line = t->line + 1;
108 column = t->column + 1;
109 }
110
111 va_list ap;
112 va_start(ap, format);
113 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
114 vfprintf(stderr, format, ap);
115 fprintf(stderr, "\n");
116 va_end(ap);
117 exit(EXIT_FAILURE);
118}
119
120static void begin_token(Tokenize *t, TokenId id) {
121 assert(!t->cur_tok);
122 t->tokens->add_one();
123 Token *token = &t->tokens->last();
124 token->start_line = t->line;
125 token->start_column = t->column;
126 token->id = id;
127 token->start_pos = t->pos;
128 t->cur_tok = token;
129}
130
131static void end_token(Tokenize *t) {
132 assert(t->cur_tok);
133 t->cur_tok->end_pos = t->pos + 1;
134
135 char *token_mem = buf_ptr(t->buf) + t->cur_tok->start_pos;
136 int token_len = t->cur_tok->end_pos - t->cur_tok->start_pos;
137
138 if (mem_eql_str(token_mem, token_len, "fn")) {
139 t->cur_tok->id = TokenIdKeywordFn;
140 } else if (mem_eql_str(token_mem, token_len, "return")) {
141 t->cur_tok->id = TokenIdKeywordReturn;
142 } else if (mem_eql_str(token_mem, token_len, "mut")) {
143 t->cur_tok->id = TokenIdKeywordMut;
144 } else if (mem_eql_str(token_mem, token_len, "const")) {
145 t->cur_tok->id = TokenIdKeywordConst;
146 }
147
148 t->cur_tok = nullptr;
149}
150
151ZigList<Token> *tokenize(Buf *buf, Buf *cur_dir_path) {
152 Tokenize t = {0};
153 t.tokens = allocate<ZigList<Token>>(1);
154 t.buf = buf;
155 t.cur_dir_path = cur_dir_path;
156 for (t.pos = 0; t.pos < buf_len(t.buf); t.pos += 1) {
157 uint8_t c = buf_ptr(t.buf)[t.pos];
158 switch (t.state) {
159 case TokenizeStateStart:
160 switch (c) {
161 case WHITESPACE:
162 break;
163 case ALPHA:
164 t.state = TokenizeStateSymbol;
165 begin_token(&t, TokenIdSymbol);
166 break;
167 case DIGIT:
168 t.state = TokenizeStateNumber;
169 begin_token(&t, TokenIdNumberLiteral);
170 break;
171 case '"':
172 begin_token(&t, TokenIdStringLiteral);
173 t.state = TokenizeStateString;
174 break;
175 case '(':
176 begin_token(&t, TokenIdLParen);
177 end_token(&t);
178 break;
179 case ')':
180 begin_token(&t, TokenIdRParen);
181 end_token(&t);
182 break;
183 case ',':
184 begin_token(&t, TokenIdComma);
185 end_token(&t);
186 break;
187 case '*':
188 begin_token(&t, TokenIdStar);
189 end_token(&t);
190 break;
191 case '{':
192 begin_token(&t, TokenIdLBrace);
193 end_token(&t);
194 break;
195 case '}':
196 begin_token(&t, TokenIdRBrace);
197 end_token(&t);
198 break;
199 case ';':
200 begin_token(&t, TokenIdSemicolon);
201 end_token(&t);
202 break;
203 case ':':
204 begin_token(&t, TokenIdColon);
205 end_token(&t);
206 break;
207 case '+':
208 begin_token(&t, TokenIdPlus);
209 end_token(&t);
210 break;
211 case '-':
212 begin_token(&t, TokenIdDash);
213 t.state = TokenizeStateSawDash;
214 break;
215 default:
216 tokenize_error(&t, "invalid character: '%c'", c);
217 }
218 break;
219 case TokenizeStateSymbol:
220 switch (c) {
221 case SYMBOL_CHAR:
222 break;
223 default:
224 t.pos -= 1;
225 end_token(&t);
226 t.state = TokenizeStateStart;
227 continue;
228 }
229 break;
230 case TokenizeStateString:
231 switch (c) {
232 case '"':
233 end_token(&t);
234 t.state = TokenizeStateStart;
235 break;
236 default:
237 break;
238 }
239 break;
240 case TokenizeStateNumber:
241 switch (c) {
242 case DIGIT:
243 break;
244 default:
245 t.pos -= 1;
246 end_token(&t);
247 t.state = TokenizeStateStart;
248 continue;
249 }
250 break;
251 case TokenizeStateSawDash:
252 switch (c) {
253 case '>':
254 t.cur_tok->id = TokenIdArrow;
255 end_token(&t);
256 t.state = TokenizeStateStart;
257 break;
258 default:
259 end_token(&t);
260 t.state = TokenizeStateStart;
261 break;
262 }
263 break;
264 }
265 if (c == '\n') {
266 t.line += 1;
267 t.column = 0;
268 } else {
269 t.column += 1;
270 }
271 }
272 // EOF
273 switch (t.state) {
274 case TokenizeStateStart:
275 break;
276 case TokenizeStateSymbol:
277 end_token(&t);
278 break;
279 case TokenizeStateString:
280 tokenize_error(&t, "unterminated string");
281 break;
282 case TokenizeStateNumber:
283 end_token(&t);
284 break;
285 case TokenizeStateSawDash:
286 end_token(&t);
287 break;
288 }
289 begin_token(&t, TokenIdEof);
290 end_token(&t);
291 assert(!t.cur_tok);
292 return t.tokens;
293}
294
295static const char * token_name(Token *token) {
296 switch (token->id) {
297 case TokenIdEof: return "EOF";
298 case TokenIdSymbol: return "Symbol";
299 case TokenIdKeywordFn: return "Fn";
300 case TokenIdKeywordConst: return "Const";
301 case TokenIdKeywordMut: return "Mut";
302 case TokenIdKeywordReturn: return "Return";
303 case TokenIdLParen: return "LParen";
304 case TokenIdRParen: return "RParen";
305 case TokenIdComma: return "Comma";
306 case TokenIdStar: return "Star";
307 case TokenIdLBrace: return "LBrace";
308 case TokenIdRBrace: return "RBrace";
309 case TokenIdStringLiteral: return "StringLiteral";
310 case TokenIdSemicolon: return "Semicolon";
311 case TokenIdNumberLiteral: return "NumberLiteral";
312 case TokenIdPlus: return "Plus";
313 case TokenIdColon: return "Colon";
314 case TokenIdArrow: return "Arrow";
315 case TokenIdDash: return "Dash";
316 }
317 return "(invalid token)";
318}
319
320void print_tokens(Buf *buf, ZigList<Token> *tokens) {
321 for (int i = 0; i < tokens->length; i += 1) {
322 Token *token = &tokens->at(i);
323 printf("%s ", token_name(token));
324 fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stdout);
325 printf("\n");
326 }
327}
328
src/tokenizer.hpp created+73
......@@ -0,0 +1,73 @@
1#ifndef ZIG_TOKENIZER_HPP
2#define ZIG_TOKENIZER_HPP
3
4#include "buffer.hpp"
5
6/*
7enum TokenId {
8 TokenIdEof,
9 TokenIdSymbol,
10 TokenIdKeywordFn,
11 TokenIdKeywordReturn,
12 TokenIdKeywordMut,
13 TokenIdKeywordConst,
14 TokenIdLParen,
15 TokenIdRParen,
16 TokenIdComma,
17 TokenIdStar,
18 TokenIdLBrace,
19 TokenIdRBrace,
20 TokenIdStringLiteral,
21 TokenIdSemicolon,
22 TokenIdNumberLiteral,
23 TokenIdPlus,
24 TokenIdColon,
25 TokenIdArrow,
26 TokenIdDash,
27};
28*/
29
30// TODO: debug delete this
31enum TokenId {
32 TokenIdStar = 0,
33 TokenIdLParen = 1,
34 TokenIdEof = 2,
35 TokenIdSymbol,
36 TokenIdKeywordFn,
37 TokenIdKeywordReturn,
38 TokenIdKeywordMut,
39 TokenIdKeywordConst,
40 TokenIdRParen,
41 TokenIdComma,
42 TokenIdLBrace,
43 TokenIdRBrace,
44 TokenIdStringLiteral,
45 TokenIdSemicolon,
46 TokenIdNumberLiteral,
47 TokenIdPlus,
48 TokenIdColon,
49 TokenIdArrow,
50 TokenIdDash,
51};
52
53struct Token {
54 TokenId id;
55 int start_pos;
56 int end_pos;
57 int start_line;
58 int start_column;
59};
60
61enum TokenizeState {
62 TokenizeStateStart,
63 TokenizeStateSymbol,
64 TokenizeStateNumber,
65 TokenizeStateString,
66 TokenizeStateSawDash,
67};
68
69ZigList<Token> *tokenize(Buf *buf, Buf *cur_dir_path);
70
71void print_tokens(Buf *buf, ZigList<Token> *tokens);
72
73#endif
src/util.hpp+8
......@@ -60,4 +60,12 @@ template<typename T>
6060static inline T clamp(T min_value, T value, T max_value) {
6161 return max(min(value, max_value), min_value);
6262}
63
64static inline bool mem_eql_str(const char *mem, size_t mem_len, const char *str) {
65 size_t str_len = strlen(str);
66 if (str_len != mem_len)
67 return false;
68 return memcmp(mem, str, mem_len) == 0;
69}
70
6371#endif