authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-04 17:15:46-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-11-04 17:15:46-07:00
log174baa49bda0d4ce1dfc99555d7e41cd86a49513
tree28eb533d48c98f0680905692a123eb21755340bf
parentc36cd9d313d5069898233188f72b0d2b5fe8f349

progress toward more complex parser gen

9 files changed, 151 insertions(+), 101 deletions(-)

CMakeLists.txt+5-4
...@@ -21,14 +21,15 @@ include_directories(...@@ -21,14 +21,15 @@ include_directories(
21)21)
2222
23set(GRAMMAR_TXT "${CMAKE_BINARY_DIR}/simple.txt")23set(GRAMMAR_TXT "${CMAKE_BINARY_DIR}/simple.txt")
24set(PARSER_CPP "${CMAKE_BINARY_DIR}/parser.cpp")24set(PARSER_GENERATED_CPP "${CMAKE_BINARY_DIR}/parser_generated.cpp")
2525
26set(ZIG_SOURCES26set(ZIG_SOURCES
27 "${CMAKE_SOURCE_DIR}/src/main.cpp"27 "${CMAKE_SOURCE_DIR}/src/main.cpp"
28 "${CMAKE_SOURCE_DIR}/src/util.cpp"28 "${CMAKE_SOURCE_DIR}/src/util.cpp"
29 "${CMAKE_SOURCE_DIR}/src/buffer.cpp"29 "${CMAKE_SOURCE_DIR}/src/buffer.cpp"
30 "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp"30 "${CMAKE_SOURCE_DIR}/src/tokenizer.cpp"
31 ${PARSER_CPP}31 "${CMAKE_SOURCE_DIR}/src/parser.cpp"
32 ${PARSER_GENERATED_CPP}
32)33)
3334
34set(PARSERGEN_SOURCES35set(PARSERGEN_SOURCES
...@@ -68,8 +69,8 @@ set_target_properties(parsergen PROPERTIES...@@ -68,8 +69,8 @@ set_target_properties(parsergen PROPERTIES
6869
6970
70add_custom_command(71add_custom_command(
71 OUTPUT ${PARSER_CPP}72 OUTPUT ${PARSER_GENERATED_CPP}
72 COMMAND parsergen ARGS ${GRAMMAR_TXT} ${PARSER_CPP}73 COMMAND parsergen ARGS ${GRAMMAR_TXT} ${PARSER_GENERATED_CPP}
73 DEPENDS ${GRAMMAR_TXT} ${PARSERGEN_SOURCES}74 DEPENDS ${GRAMMAR_TXT} ${PARSERGEN_SOURCES}
74 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}75 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
75)76)
README.md+21-15
...@@ -19,7 +19,7 @@ readable, safe, optimal, and concise code to solve any computing problem....@@ -19,7 +19,7 @@ readable, safe, optimal, and concise code to solve any computing problem.
19 * Eliminate the need for C headers (when using zig internally).19 * Eliminate the need for C headers (when using zig internally).
20 * Ability to declare dependencies as Git URLS with commit locking (can20 * Ability to declare dependencies as Git URLS with commit locking (can
21 provide a tag or sha1).21 provide a tag or sha1).
22 * Rust-style enums.22 * Tagged union enum type.
23 * Opinionated when it makes life easier.23 * Opinionated when it makes life easier.
24 - Tab character in source code is a compile error.24 - Tab character in source code is a compile error.
25 - Whitespace at the end of line is a compile error.25 - Whitespace at the end of line is a compile error.
...@@ -32,23 +32,29 @@ readable, safe, optimal, and concise code to solve any computing problem....@@ -32,23 +32,29 @@ readable, safe, optimal, and concise code to solve any computing problem.
32 * Hello, world.32 * Hello, world.
33 - Build AST33 - Build AST
34 - Code Gen34 - Code Gen
35 - Produce .o file.
36 * Produce executable file instead of .o file.
37 * Add debugging symbols.
38 * Debug/Release mode.
35 * C style comments.39 * C style comments.
36 * Unit tests.40 * Unit tests.
37 * Simple .so library41 * Simple .so library
38 * How should the Widget use case be solved? In Genesis I'm using C++ and inheritance.42 * How should the Widget use case be solved? In Genesis I'm using C++ and inheritance.
3943
40## Grammar44### Primitive Numeric Types:
4145
42```46zig | C equivalent | Description
43Root : FnDecl*47-------|--------------|-------------------------------
44FnDecl : TokenFn TokenSymbol TokenLParen list(ParamDecl, TokenComma, 0) TokenRParen (TokenArrow Type)? Block48 i8 | int8_t | signed 8-bit integer
45ParamDecl : TokenSymbol TokenColon Type49 u8 | uint8_t | unsigned 8-bit integer
46Type : TokenSymbol | PointerType50 i16 | int16_t | signed 16-bit integer
47PointerType : TokenStar (TokenConst | TokenMut) Type51 u16 | uint16_t | unsigned 16-bit integer
48Block : TokenLBrace Statement* Expression? TokenRBrace52 i32 | int32_t | signed 32-bit integer
49Statement : ExpressionStatement | ReturnStatement53 u32 | uint32_t | unsigned 32-bit integer
50ExpressionStatement : Expression TokenSemicolon54 i64 | int64_t | signed 64-bit integer
51ReturnStatement : TokenReturn Expression TokenSemicolon55 u64 | uint64_t | unsigned 64-bit integer
52Expression : TokenNumber | TokenString | FnCall56 f32 | float | 32-bit IEE754 floating point
53FnCall : TokenSymbol TokenLParen list(Expression, TokenComma, 0) TokenRParen57 f64 | double | 64-bit IEE754 floating point
54```58 f128 | long double | 128-bit IEE754 floating point
59 isize | ssize_t | signed pointer sized integer
60 usize | size_t | unsigned pointer sized integer
src/main.cpp-77
...@@ -15,7 +15,6 @@...@@ -15,7 +15,6 @@
15#include <stdio.h>15#include <stdio.h>
16#include <string.h>16#include <string.h>
17#include <stdlib.h>17#include <stdlib.h>
18#include <stdarg.h>
19#include <limits.h>18#include <limits.h>
20#include <stdint.h>19#include <stdint.h>
21#include <errno.h>20#include <errno.h>
...@@ -50,82 +49,6 @@ static Buf *fetch_file(FILE *f) {...@@ -50,82 +49,6 @@ static Buf *fetch_file(FILE *f) {
50 return buf;49 return buf;
51}50}
5251
53void ast_error(Token *token, const char *format, ...) {
54 int line = token->start_line + 1;
55 int column = token->start_column + 1;
56
57 va_list ap;
58 va_start(ap, format);
59 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
60 vfprintf(stderr, format, ap);
61 fprintf(stderr, "\n");
62 va_end(ap);
63 exit(EXIT_FAILURE);
64}
65
66static const char *node_type_str(NodeType node_type) {
67 switch (node_type) {
68 case NodeTypeRoot:
69 return "Root";
70 case NodeTypeFnDecl:
71 return "FnDecl";
72 case NodeTypeParamDecl:
73 return "ParamDecl";
74 case NodeTypeType:
75 return "Type";
76 case NodeTypePointerType:
77 return "PointerType";
78 case NodeTypeBlock:
79 return "Block";
80 case NodeTypeStatement:
81 return "Statement";
82 case NodeTypeExpressionStatement:
83 return "ExpressionStatement";
84 case NodeTypeReturnStatement:
85 return "ReturnStatement";
86 case NodeTypeExpression:
87 return "Expression";
88 case NodeTypeFnCall:
89 return "FnCall";
90 }
91 zig_panic("unreachable");
92}
93
94static void ast_print(AstNode *node, int indent) {
95 for (int i = 0; i < indent; i += 1) {
96 fprintf(stderr, " ");
97 }
98
99 switch (node->type) {
100 case NodeTypeRoot:
101 fprintf(stderr, "%s\n", node_type_str(node->type));
102 for (int i = 0; i < node->data.root.fn_decls.length; i += 1) {
103 AstNode *child = node->data.root.fn_decls.at(i);
104 ast_print(child, indent + 2);
105 }
106 break;
107 case NodeTypeFnDecl:
108 {
109 Buf *name_buf = &node->data.fn_decl.name;
110 fprintf(stderr, "%s '%s'\n", node_type_str(node->type), buf_ptr(name_buf));
111
112 for (int i = 0; i < node->data.fn_decl.params.length; i += 1) {
113 AstNode *child = node->data.fn_decl.params.at(i);
114 ast_print(child, indent + 2);
115 }
116
117 ast_print(node->data.fn_decl.return_type, indent + 2);
118
119 ast_print(node->data.fn_decl.body, indent + 2);
120
121 break;
122 }
123 default:
124 fprintf(stderr, "%s\n", node_type_str(node->type));
125 break;
126 }
127}
128
129char cur_dir[1024];52char cur_dir[1024];
13053
131int main(int argc, char **argv) {54int main(int argc, char **argv) {
src/parser.cpp created+80
...@@ -0,0 +1,80 @@
1#include "parser.hpp"
2
3#include <stdarg.h>
4#include <stdio.h>
5
6void ast_error(Token *token, const char *format, ...) {
7 int line = token->start_line + 1;
8 int column = token->start_column + 1;
9
10 va_list ap;
11 va_start(ap, format);
12 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
13 vfprintf(stderr, format, ap);
14 fprintf(stderr, "\n");
15 va_end(ap);
16 exit(EXIT_FAILURE);
17}
18
19const char *node_type_str(NodeType node_type) {
20 switch (node_type) {
21 case NodeTypeRoot:
22 return "Root";
23 case NodeTypeFnDecl:
24 return "FnDecl";
25 case NodeTypeParamDecl:
26 return "ParamDecl";
27 case NodeTypeType:
28 return "Type";
29 case NodeTypePointerType:
30 return "PointerType";
31 case NodeTypeBlock:
32 return "Block";
33 case NodeTypeStatement:
34 return "Statement";
35 case NodeTypeExpressionStatement:
36 return "ExpressionStatement";
37 case NodeTypeReturnStatement:
38 return "ReturnStatement";
39 case NodeTypeExpression:
40 return "Expression";
41 case NodeTypeFnCall:
42 return "FnCall";
43 }
44 zig_panic("unreachable");
45}
46
47void ast_print(AstNode *node, int indent) {
48 for (int i = 0; i < indent; i += 1) {
49 fprintf(stderr, " ");
50 }
51
52 switch (node->type) {
53 case NodeTypeRoot:
54 fprintf(stderr, "%s\n", node_type_str(node->type));
55 for (int i = 0; i < node->data.root.fn_decls.length; i += 1) {
56 AstNode *child = node->data.root.fn_decls.at(i);
57 ast_print(child, indent + 2);
58 }
59 break;
60 case NodeTypeFnDecl:
61 {
62 Buf *name_buf = &node->data.fn_decl.name;
63 fprintf(stderr, "%s '%s'\n", node_type_str(node->type), buf_ptr(name_buf));
64
65 for (int i = 0; i < node->data.fn_decl.params.length; i += 1) {
66 AstNode *child = node->data.fn_decl.params.at(i);
67 ast_print(child, indent + 2);
68 }
69
70 ast_print(node->data.fn_decl.return_type, indent + 2);
71
72 ast_print(node->data.fn_decl.body, indent + 2);
73
74 break;
75 }
76 default:
77 fprintf(stderr, "%s\n", node_type_str(node->type));
78 break;
79 }
80}
src/parser.hpp+5
...@@ -82,6 +82,11 @@ struct AstNode {...@@ -82,6 +82,11 @@ struct AstNode {
82__attribute__ ((format (printf, 2, 3)))82__attribute__ ((format (printf, 2, 3)))
83void ast_error(Token *token, const char *format, ...);83void ast_error(Token *token, const char *format, ...);
8484
85// This function is provided by generated code, generated by parsergen.cpp
85AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens);86AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens);
8687
88const char *node_type_str(NodeType node_type);
89
90void ast_print(AstNode *node, int indent);
91
87#endif92#endif
src/parsergen.cpp+25-2
...@@ -190,12 +190,17 @@ struct RuleNode {...@@ -190,12 +190,17 @@ struct RuleNode {
190enum ParserStateType {190enum ParserStateType {
191 ParserStateTypeError,191 ParserStateTypeError,
192 ParserStateTypeOk,192 ParserStateTypeOk,
193 ParserStateTypeCapture,
193};194};
194195
195struct ParserStateError {196struct ParserStateError {
196 Buf *msg;197 Buf *msg;
197};198};
198199
200struct ParserStateCapture {
201 Buf *body;
202};
203
199struct ParserState {204struct ParserState {
200 ParserStateType type;205 ParserStateType type;
201 // One for each token ID.206 // One for each token ID.
...@@ -203,6 +208,7 @@ struct ParserState {...@@ -203,6 +208,7 @@ struct ParserState {
203 int index;208 int index;
204 union {209 union {
205 ParserStateError error;210 ParserStateError error;
211 ParserStateCapture capture;
206 };212 };
207};213};
208214
...@@ -278,6 +284,8 @@ static void gen(Gen *g, RuleNode *node) {...@@ -278,6 +284,8 @@ static void gen(Gen *g, RuleNode *node) {
278 RuleNode *child = node->tuple.children.at(i);284 RuleNode *child = node->tuple.children.at(i);
279 gen(g, child);285 gen(g, child);
280 }286 }
287 g->cur_state->type = ParserStateTypeCapture;
288 g->cur_state->capture.body = &node->tuple.body;
281 }289 }
282 break;290 break;
283 case RuleNodeTypeMany:291 case RuleNodeTypeMany:
...@@ -598,7 +606,8 @@ int main(int argc, char **argv) {...@@ -598,7 +606,8 @@ int main(int argc, char **argv) {
598 g.cur_state = create_state(&g, ParserStateTypeOk);606 g.cur_state = create_state(&g, ParserStateTypeOk);
599 gen(&g, g.root);607 gen(&g, g.root);
600608
601 fprintf(out_f, "/* This file is auto-generated by parsergen.cpp */\n");609 fprintf(out_f, "/* This file is generated by parsergen.cpp */\n");
610 fprintf(out_f, "\n");
602 fprintf(out_f, "#include \"src/parser.hpp\"\n");611 fprintf(out_f, "#include \"src/parser.hpp\"\n");
603 fprintf(out_f, "#include <stdio.h>\n");612 fprintf(out_f, "#include <stdio.h>\n");
604613
...@@ -616,6 +625,17 @@ int main(int argc, char **argv) {...@@ -616,6 +625,17 @@ int main(int argc, char **argv) {
616 fprintf(out_f, "static_assert(TokenId%s == %d, \"wrong token id\");\n",625 fprintf(out_f, "static_assert(TokenId%s == %d, \"wrong token id\");\n",
617 buf_ptr(&token->name), token->id);626 buf_ptr(&token->name), token->id);
618 }627 }
628 fprintf(out_f, "\n");
629
630 /* TODO
631 fprintf(out_f, "struct ParserGenNode{\n");
632 fprintf(out_f, " union {\n");
633 fprintf(out_f, " [%d];\n", biggest_tuple_len);
634 fprintf(out_f, " Token *token;\n");
635 fprintf(out_f, " };\n");
636 fprintf(out_f, "};\n");
637 fprintf(out_f, "\n");
638 */
619639
620 fprintf(out_f, "AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens) {\n");640 fprintf(out_f, "AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens) {\n");
621641
...@@ -644,7 +664,6 @@ int main(int argc, char **argv) {...@@ -644,7 +664,6 @@ int main(int argc, char **argv) {
644 for (int i = 0; i < g.transition_table.length; i += 1) {664 for (int i = 0; i < g.transition_table.length; i += 1) {
645 ParserState *state = g.transition_table.at(i);665 ParserState *state = g.transition_table.at(i);
646 fprintf(out_f, " case %d:\n", i);666 fprintf(out_f, " case %d:\n", i);
647 fprintf(out_f, " fprintf(stderr, \"state = %%d\\n\", state);\n");
648 switch (state->type) {667 switch (state->type) {
649 case ParserStateTypeError:668 case ParserStateTypeError:
650 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(state->error.msg));669 fprintf(out_f, " ast_error(token, \"%s\");\n", buf_ptr(state->error.msg));
...@@ -655,6 +674,10 @@ int main(int argc, char **argv) {...@@ -655,6 +674,10 @@ int main(int argc, char **argv) {
655 state->index, g.transition_table.length);674 state->index, g.transition_table.length);
656 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);675 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
657 break;676 break;
677 case ParserStateTypeCapture:
678 // TODO fprintf(out_f, " %s\n", buf_ptr(state->capture.body));
679 fprintf(out_f, " state = transition[%d][token->id];\n", state->index);
680 break;
658 }681 }
659 fprintf(out_f, " break;\n");682 fprintf(out_f, " break;\n");
660 }683 }
src/tokenizer.cpp+7
...@@ -1,3 +1,10 @@...@@ -1,3 +1,10 @@
1/*
2 * Copyright (c) 2015 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
1#include "tokenizer.hpp"8#include "tokenizer.hpp"
2#include "util.hpp"9#include "util.hpp"
310
src/tokenizer.hpp+7
...@@ -1,3 +1,10 @@...@@ -1,3 +1,10 @@
1/*
2 * Copyright (c) 2015 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
1#ifndef ZIG_TOKENIZER_HPP8#ifndef ZIG_TOKENIZER_HPP
2#define ZIG_TOKENIZER_HPP9#define ZIG_TOKENIZER_HPP
310
test/hello.zig+1-3
...@@ -1,6 +1,4 @@...@@ -1,6 +1,4 @@
11fn main(argc: i32, argv: *mut u8) -> i32 {
2
3fn main(argc: isize, argv: *mut u8) -> isize {
4 puts("Hello, world!\n");2 puts("Hello, world!\n");
5 return 0;3 return 0;
6}4}