authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-23 21:30:12-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-23 21:30:12-07:00
log1b24f4c73c7f61d123eb8f0c4e90b267d77ec596
tree89b093909b08c92299c8823772e17394f23c6d87
parent6b911f1e94b77730b78493183aa85296bb0fdd76

parsing hello.zig example with recursive descent

that was easy

7 files changed, 378 insertions(+), 103 deletions(-)

README.md+28
......@@ -69,3 +69,31 @@ zig | C equivalent | Description
6969 f128 | long double | 128-bit IEE754 floating point
7070 isize | ssize_t | signed pointer sized integer
7171 usize | size_t | unsigned pointer sized integer
72
73### Grammar
74
75```
76Root : many(FnDecl) token(EOF);
77
78FnDecl : token(Fn) token(Symbol) ParamDeclList option(token(Arrow) Type) Block;
79
80ParamDeclList : token(LParen) list(ParamDecl, token(Comma)) token(RParen);
81
82ParamDecl : token(Symbol) token(Colon) Type;
83
84Type : token(Symbol) | PointerType;
85
86PointerType : token(Star) token(Const) Type | token(Star) token(Mut) Type;
87
88Block : token(LBrace) many(Statement) token(RBrace);
89
90Statement : ExpressionStatement | ReturnStatement ;
91
92ExpressionStatement : Expression token(Semicolon) ;
93
94ReturnStatement : token(Return) Expression token(Semicolon) ;
95
96Expression : token(Number) | token(String) | FnCall ;
97
98FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) ;
99```
src/grammar.txt deleted-57
......@@ -1,57 +0,0 @@
1Root<node> : many(FnDecl) token(EOF) {
2 $$ = ast_create_root($1);
3};
4
5FnDecl<node> : token(Fn) token(Symbol) token(LParen) list(ParamDecl, token(Comma)) token(RParen) token(Arrow) Type Block {
6 $$ = ast_create_fn_decl($2, $4, $7, $8);
7} | token(Fn) token(Symbol) token(LParen) list(ParamDecl, token(Comma)) token(RParen) Block {
8 $$ = ast_create_void_fn_decl($2, $4, $6);
9};
10
11ParamDecl<node> : token(Symbol) token(Colon) Type {
12 $$ = ast_create_param_decl($1, $2);
13};
14
15Type<node> : token(Symbol) {
16 $$ = ast_create_symbol_type($1);
17} | PointerType {
18 $$ = $1;
19};
20
21PointerType<node> : token(Star) token(Const) Type {
22 $$ = ast_create_pointer_type($2, $3);
23} | token(Star) token(Mut) Type {
24 $$ = ast_create_pointer_type($2, $3);
25};
26
27Block<node> : token(LBrace) many(Statement) Expression token(RBrace) {
28 $$ = ast_create_expr_block($2, $3);
29} | token(LBrace) many(Statement) token(RBrace) {
30 $$ = ast_create_block($2);
31};
32
33Statement<node> : ExpressionStatement {
34 $$ = $1;
35} | ReturnStatement {
36 $$ = $1;
37};
38
39ExpressionStatement<node> : Expression token(Semicolon) {
40 $$ = ast_create_expression_statement($1);
41};
42
43ReturnStatement<node> : token(Return) Expression token(Semicolon) {
44 $$ = ast_create_return_statement($2);
45};
46
47Expression<node> : token(Number) {
48 $$ = ast_create_number($1);
49} | token(String) {
50 $$ = ast_create_string($1);
51} | FnCall {
52 $$ = $1;
53};
54
55FnCall<node> : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) {
56 $$ = ast_create_fn_call($1, $3);
57};
src/main.cpp+4-2
......@@ -89,6 +89,8 @@ static int build(const char *arg0, const char *in_file, const char *out_file, Zi
8989
9090 AstNode *root = ast_parse(in_data, tokens);
9191 assert(root);
92 fprintf(stderr, "\nAST:\n");
93 fprintf(stderr, "------\n");
9294 ast_print(root, 0);
9395
9496
......@@ -135,7 +137,7 @@ int main(int argc, char **argv) {
135137 } else {
136138 switch (cmd) {
137139 case CmdNone:
138 zig_panic("unreachable");
140 zig_unreachable();
139141 case CmdBuild:
140142 if (!in_file) {
141143 in_file = arg;
......@@ -154,6 +156,6 @@ int main(int argc, char **argv) {
154156 return build(arg0, in_file, out_file, &include_paths);
155157 }
156158
157 zig_panic("unreachable");
159 zig_unreachable();
158160}
159161
src/parser.cpp+302-9
......@@ -41,7 +41,7 @@ const char *node_type_str(NodeType node_type) {
4141 case NodeTypeFnCall:
4242 return "FnCall";
4343 }
44 zig_panic("unreachable");
44 zig_unreachable();
4545}
4646
4747void ast_print(AstNode *node, int indent) {
......@@ -82,28 +82,321 @@ void ast_print(AstNode *node, int indent) {
8282struct ParseContext {
8383 Buf *buf;
8484 AstNode *root;
85 ZigList<Token> *tokens;
8586};
8687
87AstNode *ast_create_root(void) {
88 zig_panic("TODO create root");
88static AstNode *ast_create_node(NodeType type) {
89 AstNode *node = allocate<AstNode>(1);
90 node->type = type;
91 return node;
8992}
9093
91void ast_invalid_token_error(Buf *buf, Token *token) {
94static void ast_buf_from_token(ParseContext *pc, Token *token, Buf *buf) {
95 buf_init_from_mem(buf, buf_ptr(pc->buf) + token->start_pos, token->end_pos - token->start_pos);
96}
97
98static void ast_invalid_token_error(ParseContext *pc, Token *token) {
9299 Buf token_value = {0};
93 buf_init_from_mem(&token_value, buf_ptr(buf) + token->start_pos, token->end_pos - token->start_pos);
100 ast_buf_from_token(pc, token, &token_value);
94101 ast_error(token, "invalid token: '%s'", buf_ptr(&token_value));
95102}
96103
97void ast_parse_fn_decls(ParseContext *pc, ZigList<AstNode *> *fn_decls) {
98 zig_panic("TODO parse fn decls");
104static AstNode *ast_parse_expression(ParseContext *pc, int token_index, int *new_token_index);
105
106
107static void ast_expect_token(ParseContext *pc, Token *token, TokenId token_id) {
108 if (token->id != token_id) {
109 ast_invalid_token_error(pc, token);
110 }
111}
112
113/*
114Type : token(Symbol) | PointerType;
115PointerType : token(Star) token(Const) Type | token(Star) token(Mut) Type;
116*/
117static AstNode *ast_parse_type(ParseContext *pc, int token_index, int *new_token_index) {
118 AstNode *node = ast_create_node(NodeTypeType);
119
120 Token *token = &pc->tokens->at(token_index);
121 token_index += 1;
122
123 if (token->id == TokenIdSymbol) {
124 node->data.type.type = AstNodeTypeTypePrimitive;
125 ast_buf_from_token(pc, token, &node->data.type.primitive_name);
126 } else if (token->id == TokenIdStar) {
127 Token *const_or_mut = &pc->tokens->at(token_index);
128 token_index += 1;
129 if (const_or_mut->id == TokenIdKeywordMut) {
130 node->data.type.is_const = false;
131 } else if (const_or_mut->id == TokenIdKeywordConst) {
132 node->data.type.is_const = true;
133 } else {
134 ast_invalid_token_error(pc, const_or_mut);
135 }
136
137 node->data.type.child_type = ast_parse_type(pc, token_index, &token_index);
138 } else {
139 ast_invalid_token_error(pc, token);
140 }
141
142 *new_token_index = token_index;
143 return node;
144}
145
146/*
147ParamDecl<node> : token(Symbol) token(Colon) Type {
148};
149*/
150static AstNode *ast_parse_param_decl(ParseContext *pc, int token_index, int *new_token_index) {
151 AstNode *node = ast_create_node(NodeTypeParamDecl);
152
153 Token *param_name = &pc->tokens->at(token_index);
154 token_index += 1;
155 ast_expect_token(pc, param_name, TokenIdSymbol);
156
157 ast_buf_from_token(pc, param_name, &node->data.param_decl.name);
158
159 Token *colon = &pc->tokens->at(token_index);
160 token_index += 1;
161 ast_expect_token(pc, colon, TokenIdColon);
162
163 node->data.param_decl.type = ast_parse_type(pc, token_index, &token_index);
164
165 *new_token_index = token_index;
166 return node;
167}
168
169
170static void ast_parse_param_decl_list(ParseContext *pc, int token_index, int *new_token_index,
171 ZigList<AstNode *> *params)
172{
173 Token *l_paren = &pc->tokens->at(token_index);
174 token_index += 1;
175 ast_expect_token(pc, l_paren, TokenIdLParen);
176
177 Token *token = &pc->tokens->at(token_index);
178 if (token->id == TokenIdRParen) {
179 token_index += 1;
180 *new_token_index = token_index;
181 return;
182 }
183
184 for (;;) {
185 AstNode *param_decl_node = ast_parse_param_decl(pc, token_index, &token_index);
186 params->append(param_decl_node);
187
188 Token *token = &pc->tokens->at(token_index);
189 token_index += 1;
190 if (token->id == TokenIdRParen) {
191 *new_token_index = token_index;
192 return;
193 } else {
194 ast_expect_token(pc, token, TokenIdComma);
195 }
196 }
197 zig_unreachable();
198}
199
200static void ast_parse_fn_call_param_list(ParseContext *pc, int token_index, int *new_token_index,
201 ZigList<AstNode*> *params)
202{
203 Token *l_paren = &pc->tokens->at(token_index);
204 token_index += 1;
205 ast_expect_token(pc, l_paren, TokenIdLParen);
206
207 Token *token = &pc->tokens->at(token_index);
208 if (token->id == TokenIdRParen) {
209 token_index += 1;
210 *new_token_index = token_index;
211 return;
212 }
213
214 for (;;) {
215 AstNode *expr = ast_parse_expression(pc, token_index, &token_index);
216 params->append(expr);
217
218 Token *token = &pc->tokens->at(token_index);
219 token_index += 1;
220 if (token->id == TokenIdRParen) {
221 *new_token_index = token_index;
222 return;
223 } else {
224 ast_expect_token(pc, token, TokenIdComma);
225 }
226 }
227 zig_unreachable();
228}
229
230/*
231FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) ;
232*/
233static AstNode *ast_parse_fn_call(ParseContext *pc, int token_index, int *new_token_index) {
234 AstNode *node = ast_create_node(NodeTypeFnCall);
235
236 Token *fn_name = &pc->tokens->at(token_index);
237 token_index += 1;
238 ast_expect_token(pc, fn_name, TokenIdSymbol);
239
240 ast_buf_from_token(pc, fn_name, &node->data.fn_call.name);
241
242 ast_parse_fn_call_param_list(pc, token_index, &token_index, &node->data.fn_call.params);
243
244 *new_token_index = token_index;
245 return node;
246}
247
248static AstNode *ast_parse_expression(ParseContext *pc, int token_index, int *new_token_index) {
249 AstNode *node = ast_create_node(NodeTypeExpression);
250
251 Token *token = &pc->tokens->at(token_index);
252 if (token->id == TokenIdSymbol) {
253 node->data.expression.type = AstNodeExpressionTypeFnCall;
254 node->data.expression.data.fn_call = ast_parse_fn_call(pc, token_index, &token_index);
255 } else if (token->id == TokenIdNumberLiteral) {
256 node->data.expression.type = AstNodeExpressionTypeNumber;
257 ast_buf_from_token(pc, token, &node->data.expression.data.number);
258 token_index += 1;
259 } else if (token->id == TokenIdStringLiteral) {
260 node->data.expression.type = AstNodeExpressionTypeString;
261 ast_buf_from_token(pc, token, &node->data.expression.data.string);
262 token_index += 1;
263 } else {
264 ast_invalid_token_error(pc, token);
265 }
266
267 *new_token_index = token_index;
268 return node;
269}
270
271/*
272Statement : ExpressionStatement | ReturnStatement ;
273
274ExpressionStatement : Expression token(Semicolon) ;
275
276ReturnStatement : token(Return) Expression token(Semicolon) ;
277
278Expression : token(Number) | token(String) | FnCall ;
279
280FnCall : token(Symbol) token(LParen) list(Expression, token(Comma)) token(RParen) ;
281*/
282static AstNode *ast_parse_statement(ParseContext *pc, int token_index, int *new_token_index) {
283 AstNode *node = ast_create_node(NodeTypeStatement);
284
285 Token *token = &pc->tokens->at(token_index);
286 if (token->id == TokenIdKeywordReturn) {
287 token_index += 1;
288 node->data.statement.type = AstNodeStatementTypeReturn;
289 node->data.statement.data.retrn.expression = ast_parse_expression(pc, token_index, &token_index);
290
291 Token *semicolon = &pc->tokens->at(token_index);
292 token_index += 1;
293 ast_expect_token(pc, semicolon, TokenIdSemicolon);
294 } else if (token->id == TokenIdSymbol ||
295 token->id == TokenIdStringLiteral ||
296 token->id == TokenIdNumberLiteral)
297 {
298 node->data.statement.type = AstNodeStatementTypeExpression;
299 node->data.statement.data.expr.expression = ast_parse_expression(pc, token_index, &token_index);
300
301 Token *semicolon = &pc->tokens->at(token_index);
302 token_index += 1;
303 ast_expect_token(pc, semicolon, TokenIdSemicolon);
304 } else {
305 ast_invalid_token_error(pc, token);
306 }
307
308 *new_token_index = token_index;
309 return node;
310}
311
312/*
313Block : token(LBrace) many(Statement) token(RBrace);
314*/
315static AstNode *ast_parse_block(ParseContext *pc, int token_index, int *new_token_index) {
316 AstNode *node = ast_create_node(NodeTypeBlock);
317
318 Token *l_brace = &pc->tokens->at(token_index);
319 token_index += 1;
320 ast_expect_token(pc, l_brace, TokenIdLBrace);
321
322 for (;;) {
323 Token *token = &pc->tokens->at(token_index);
324 if (token->id == TokenIdRBrace) {
325 token_index += 1;
326 *new_token_index = token_index;
327 return node;
328 } else {
329 AstNode *statement_node = ast_parse_statement(pc, token_index, &token_index);
330 node->data.block.statements.append(statement_node);
331 }
332 }
333 zig_unreachable();
334}
335
336/*
337FnDecl : token(Fn) token(Symbol) ParamDeclList option(token(Arrow) Type) Block;
338*/
339static AstNode *ast_parse_fn_decl(ParseContext *pc, int token_index, int *new_token_index) {
340 AstNode *node = ast_create_node(NodeTypeFnDecl);
341
342 Token *fn_token = &pc->tokens->at(token_index);
343 token_index += 1;
344 ast_expect_token(pc, fn_token, TokenIdKeywordFn);
345
346 Token *fn_name = &pc->tokens->at(token_index);
347 token_index += 1;
348 ast_expect_token(pc, fn_name, TokenIdSymbol);
349
350 ast_buf_from_token(pc, fn_name, &node->data.fn_decl.name);
351
352
353 ast_parse_param_decl_list(pc, token_index, &token_index, &node->data.fn_decl.params);
354
355 Token *arrow = &pc->tokens->at(token_index);
356 token_index += 1;
357 if (arrow->id == TokenIdArrow) {
358 node->data.fn_decl.return_type = ast_parse_type(pc, token_index, &token_index);
359 } else if (arrow->id == TokenIdLBrace) {
360 node->data.fn_decl.return_type = nullptr;
361 } else {
362 ast_invalid_token_error(pc, arrow);
363 }
364
365 node->data.fn_decl.body = ast_parse_block(pc, token_index, &token_index);
366
367 *new_token_index = token_index;
368 return node;
369}
370
371
372static void ast_parse_fn_decl_list(ParseContext *pc, int token_index, ZigList<AstNode *> *fn_decls,
373 int *new_token_index)
374{
375 for (;;) {
376 Token *token = &pc->tokens->at(token_index);
377 if (token->id == TokenIdKeywordFn) {
378 AstNode *fn_decl_node = ast_parse_fn_decl(pc, token_index, &token_index);
379 fn_decls->append(fn_decl_node);
380 } else {
381 *new_token_index = token_index;
382 return;
383 }
384 }
385 zig_unreachable();
99386}
100387
101388AstNode *ast_parse(Buf *buf, ZigList<Token> *tokens) {
102389 ParseContext pc = {0};
103390 pc.buf = buf;
104 pc.root = ast_create_root();
391 pc.root = ast_create_node(NodeTypeRoot);
392 pc.tokens = tokens;
105393
106 ast_parse_fn_decls(&pc, &pc.root->data.root.fn_decls);
394 int new_token_index;
395 ast_parse_fn_decl_list(&pc, 0, &pc.root->data.root.fn_decls, &new_token_index);
396
397 if (new_token_index != tokens->length - 1) {
398 ast_invalid_token_error(&pc, &tokens->at(new_token_index));
399 }
107400
108401 return pc.root;
109402}
src/parser.hpp+38-10
......@@ -44,20 +44,49 @@ enum AstNodeTypeType {
4444
4545struct AstNodeType {
4646 AstNodeTypeType type;
47 AstNode *child;
47 Buf primitive_name;
48 AstNode *child_type;
49 bool is_const;
4850};
4951
50struct AstNodePointerType {
51 AstNode *const_or_mut;
52 AstNode *type;
52struct AstNodeBlock {
53 ZigList<AstNode *> statements;
5354};
5455
55struct AstNodeBlock {
56 ZigList<AstNode *> expressions;
56enum AstNodeStatementType {
57 AstNodeStatementTypeExpression,
58 AstNodeStatementTypeReturn,
59};
60
61struct AstNodeStatementExpression {
62 AstNode *expression;
63};
64
65struct AstNodeStatementReturn {
66 AstNode *expression;
67};
68
69struct AstNodeStatement {
70 AstNodeStatementType type;
71 union {
72 AstNodeStatementExpression expr;
73 AstNodeStatementReturn retrn;
74 } data;
75};
76
77enum AstNodeExpressionType {
78 AstNodeExpressionTypeNumber,
79 AstNodeExpressionTypeString,
80 AstNodeExpressionTypeFnCall,
5781};
5882
5983struct AstNodeExpression {
60 AstNode *child;
84 AstNodeExpressionType type;
85 union {
86 Buf number;
87 Buf string;
88 AstNode *fn_call;
89 } data;
6190};
6291
6392struct AstNodeFnCall {
......@@ -74,13 +103,14 @@ struct AstNode {
74103 AstNodeType type;
75104 AstNodeParamDecl param_decl;
76105 AstNodeBlock block;
106 AstNodeStatement statement;
77107 AstNodeExpression expression;
78108 AstNodeFnCall fn_call;
79109 } data;
80110};
81111
82112__attribute__ ((format (printf, 2, 3)))
83void ast_error(Token *token, const char *format, ...);
113void ast_token_error(Token *token, const char *format, ...);
84114void ast_invalid_token_error(Buf *buf, Token *token);
85115
86116
......@@ -91,6 +121,4 @@ const char *node_type_str(NodeType node_type);
91121
92122void ast_print(AstNode *node, int indent);
93123
94AstNode *ast_create_root(void);
95
96124#endif
src/tokenizer.hpp-25
......@@ -10,7 +10,6 @@
1010
1111#include "buffer.hpp"
1212
13/*
1413enum TokenId {
1514 TokenIdEof,
1615 TokenIdSymbol,
......@@ -32,30 +31,6 @@ enum TokenId {
3231 TokenIdArrow,
3332 TokenIdDash,
3433};
35*/
36
37// TODO: debug delete this
38enum TokenId {
39 TokenIdLParen = 0,
40 TokenIdRParen = 1,
41 TokenIdEof = 2,
42 TokenIdStar = 3,
43 TokenIdPlus = 4,
44 TokenIdSymbol,
45 TokenIdKeywordFn,
46 TokenIdKeywordReturn,
47 TokenIdKeywordMut,
48 TokenIdKeywordConst,
49 TokenIdComma,
50 TokenIdLBrace,
51 TokenIdRBrace,
52 TokenIdStringLiteral,
53 TokenIdSemicolon,
54 TokenIdNumberLiteral,
55 TokenIdColon,
56 TokenIdArrow,
57 TokenIdDash,
58};
5934
6035struct Token {
6136 TokenId id;
src/util.hpp+6
......@@ -20,6 +20,12 @@ void zig_panic(const char *format, ...)
2020 __attribute__ ((noreturn))
2121 __attribute__ ((format (printf, 1, 2)));
2222
23__attribute__((cold))
24__attribute__ ((noreturn))
25static inline void zig_unreachable(void) {
26 zig_panic("unreachable");
27}
28
2329template<typename T>
2430__attribute__((malloc)) static inline T *allocate_nonzero(size_t count) {
2531 T *ptr = reinterpret_cast<T*>(malloc(count * sizeof(T)));