authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-12-01 00:50:11-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-12-01 00:50:11-07:00
log257cf09472ce5f4a51bf39808e119717fa0e4280
tree722cbc737134b1aa03fafac400f680f20007308a
parent31cf43de54c012c3b6f4fa7a516e2aac0ae18b56

colored error messages that tell the source file

16 files changed, 240 insertions(+), 113 deletions(-)

CMakeLists.txt+1
...@@ -31,6 +31,7 @@ set(ZIG_SOURCES...@@ -31,6 +31,7 @@ set(ZIG_SOURCES
31 "${CMAKE_SOURCE_DIR}/src/main.cpp"31 "${CMAKE_SOURCE_DIR}/src/main.cpp"
32 "${CMAKE_SOURCE_DIR}/src/os.cpp"32 "${CMAKE_SOURCE_DIR}/src/os.cpp"
33 "${CMAKE_SOURCE_DIR}/src/util.cpp"33 "${CMAKE_SOURCE_DIR}/src/util.cpp"
34 "${CMAKE_SOURCE_DIR}/src/errmsg.cpp"
34 "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp"35 "${CMAKE_SOURCE_DIR}/src/zig_llvm.cpp"
35)36)
3637
src/analyze.cpp+11-7
...@@ -12,13 +12,17 @@...@@ -12,13 +12,17 @@
12#include "os.hpp"12#include "os.hpp"
1313
14static void add_node_error(CodeGen *g, AstNode *node, Buf *msg) {14static void add_node_error(CodeGen *g, AstNode *node, Buf *msg) {
15 g->errors.add_one();15 ErrorMsg *err = allocate<ErrorMsg>(1);
16 ErrorMsg *last_msg = &g->errors.last();16 err->line_start = node->line;
17 last_msg->line_start = node->line;17 err->column_start = node->column;
18 last_msg->column_start = node->column;18 err->line_end = -1;
19 last_msg->line_end = -1;19 err->column_end = -1;
20 last_msg->column_end = -1;20 err->msg = msg;
21 last_msg->msg = msg;21 err->path = node->owner->path;
22 err->source = node->owner->source_code;
23 err->line_offsets = node->owner->line_offsets;
24
25 g->errors.append(err);
22}26}
2327
24static int parse_version_string(Buf *buf, int *major, int *minor, int *patch) {28static int parse_version_string(Buf *buf, int *major, int *minor, int *patch) {
src/buffer.cpp+10-4
...@@ -3,9 +3,8 @@...@@ -3,9 +3,8 @@
3#include <stdlib.h>3#include <stdlib.h>
4#include <stdio.h>4#include <stdio.h>
55
6Buf *buf_sprintf(const char *format, ...) {6Buf *buf_vprintf(const char *format, va_list ap) {
7 va_list ap, ap2;7 va_list ap2;
8 va_start(ap, format);
9 va_copy(ap2, ap);8 va_copy(ap2, ap);
109
11 int len1 = vsnprintf(nullptr, 0, format, ap);10 int len1 = vsnprintf(nullptr, 0, format, ap);
...@@ -19,11 +18,18 @@ Buf *buf_sprintf(const char *format, ...) {...@@ -19,11 +18,18 @@ Buf *buf_sprintf(const char *format, ...) {
19 assert(len2 == len1);18 assert(len2 == len1);
2019
21 va_end(ap2);20 va_end(ap2);
22 va_end(ap);
2321
24 return buf;22 return buf;
25}23}
2624
25Buf *buf_sprintf(const char *format, ...) {
26 va_list ap;
27 va_start(ap, format);
28 Buf *result = buf_vprintf(format, ap);
29 va_end(ap);
30 return result;
31}
32
27void buf_appendf(Buf *buf, const char *format, ...) {33void buf_appendf(Buf *buf, const char *format, ...) {
28 assert(buf->list.length);34 assert(buf->list.length);
29 va_list ap, ap2;35 va_list ap, ap2;
src/buffer.hpp+2
...@@ -13,6 +13,7 @@...@@ -13,6 +13,7 @@
13#include <assert.h>13#include <assert.h>
14#include <stdint.h>14#include <stdint.h>
15#include <ctype.h>15#include <ctype.h>
16#include <stdarg.h>
1617
17#define BUF_INIT {{0}}18#define BUF_INIT {{0}}
1819
...@@ -24,6 +25,7 @@ struct Buf {...@@ -24,6 +25,7 @@ struct Buf {
2425
25Buf *buf_sprintf(const char *format, ...)26Buf *buf_sprintf(const char *format, ...)
26 __attribute__ ((format (printf, 1, 2)));27 __attribute__ ((format (printf, 1, 2)));
28Buf *buf_vprintf(const char *format, va_list ap);
2729
28static inline int buf_len(Buf *buf) {30static inline int buf_len(Buf *buf) {
29 assert(buf->list.length);31 assert(buf->list.length);
src/codegen.cpp+25-8
...@@ -13,6 +13,7 @@...@@ -13,6 +13,7 @@
13#include "error.hpp"13#include "error.hpp"
14#include "semantic_info.hpp"14#include "semantic_info.hpp"
15#include "analyze.hpp"15#include "analyze.hpp"
16#include "errmsg.hpp"
1617
17#include <stdio.h>18#include <stdio.h>
18#include <errno.h>19#include <errno.h>
...@@ -678,24 +679,42 @@ static ImportTableEntry *codegen_add_code(CodeGen *g, Buf *source_path, Buf *sou...@@ -678,24 +679,42 @@ static ImportTableEntry *codegen_add_code(CodeGen *g, Buf *source_path, Buf *sou
678 fprintf(stderr, "---------\n");679 fprintf(stderr, "---------\n");
679 }680 }
680681
681 ZigList<Token> *tokens = tokenize(source_code);682 Tokenization tokenization = {0};
683 tokenize(source_code, &tokenization);
684
685 if (tokenization.err) {
686 ErrorMsg *err = allocate<ErrorMsg>(1);
687 err->line_start = tokenization.err_line;
688 err->column_start = tokenization.err_column;
689 err->line_end = -1;
690 err->column_end = -1;
691 err->msg = tokenization.err;
692 err->path = source_path;
693 err->source = source_code;
694 err->line_offsets = tokenization.line_offsets;
695
696 print_err_msg(err);
697 exit(1);
698 }
682699
683 if (g->verbose) {700 if (g->verbose) {
684 print_tokens(source_code, tokens);701 print_tokens(source_code, tokenization.tokens);
685702
686 fprintf(stderr, "\nAST:\n");703 fprintf(stderr, "\nAST:\n");
687 fprintf(stderr, "------\n");704 fprintf(stderr, "------\n");
688 }705 }
689706
690 ImportTableEntry *import_entry = allocate<ImportTableEntry>(1);707 ImportTableEntry *import_entry = allocate<ImportTableEntry>(1);
708 import_entry->source_code = source_code;
709 import_entry->line_offsets = tokenization.line_offsets;
710 import_entry->path = source_path;
691 import_entry->fn_table.init(32);711 import_entry->fn_table.init(32);
692 import_entry->root = ast_parse(source_code, tokens);712 import_entry->root = ast_parse(source_code, tokenization.tokens, import_entry);
693 assert(import_entry->root);713 assert(import_entry->root);
694 if (g->verbose) {714 if (g->verbose) {
695 ast_print(import_entry->root, 0);715 ast_print(import_entry->root, 0);
696 }716 }
697717
698 import_entry->path = source_path;
699 import_entry->di_file = LLVMZigCreateFile(g->dbuilder, buf_ptr(&basename), buf_ptr(&dirname));718 import_entry->di_file = LLVMZigCreateFile(g->dbuilder, buf_ptr(&basename), buf_ptr(&dirname));
700 g->import_table.put(source_path, import_entry);719 g->import_table.put(source_path, import_entry);
701720
...@@ -736,10 +755,8 @@ void codegen_add_root_code(CodeGen *g, Buf *source_path, Buf *source_code) {...@@ -736,10 +755,8 @@ void codegen_add_root_code(CodeGen *g, Buf *source_path, Buf *source_code) {
736 }755 }
737 } else {756 } else {
738 for (int i = 0; i < g->errors.length; i += 1) {757 for (int i = 0; i < g->errors.length; i += 1) {
739 ErrorMsg *err = &g->errors.at(i);758 ErrorMsg *err = g->errors.at(i);
740 fprintf(stderr, "Error: Line %d, column %d: %s\n",759 print_err_msg(err);
741 err->line_start + 1, err->column_start + 1,
742 buf_ptr(err->msg));
743 }760 }
744 exit(1);761 exit(1);
745 }762 }
src/codegen.hpp-9
...@@ -20,15 +20,6 @@ enum OutType {...@@ -20,15 +20,6 @@ enum OutType {
20};20};
2121
2222
23struct ErrorMsg {
24 int line_start;
25 int column_start;
26 int line_end;
27 int column_end;
28 Buf *msg;
29};
30
31
32CodeGen *codegen_create(Buf *root_source_dir);23CodeGen *codegen_create(Buf *root_source_dir);
3324
34enum CodeGenBuildType {25enum CodeGenBuildType {
src/errmsg.cpp created+38
...@@ -0,0 +1,38 @@
1#include "errmsg.hpp"
2#include "os.hpp"
3
4#include <stdio.h>
5
6#define RED "\x1b[31;1m"
7#define WHITE "\x1b[37;1m"
8#define GREEN "\x1b[32;1m"
9#define RESET "\x1b[0m"
10
11void print_err_msg(ErrorMsg *err) {
12 if (os_stderr_tty()) {
13 fprintf(stderr, WHITE "%s:%d:%d: " RED "error:" WHITE " %s" RESET "\n",
14 buf_ptr(err->path),
15 err->line_start + 1, err->column_start + 1,
16 buf_ptr(err->msg));
17
18 assert(err->source);
19 assert(err->line_offsets);
20
21 int line_start_offset = err->line_offsets->at(err->line_start);
22 int line_end_offset = err->line_offsets->at(err->line_start + 1);
23
24 fwrite(buf_ptr(err->source) + line_start_offset, 1, line_end_offset - line_start_offset - 1, stderr);
25 fprintf(stderr, "\n");
26 for (int i = 0; i < err->column_start; i += 1) {
27 fprintf(stderr, " ");
28 }
29 fprintf(stderr, GREEN "^" RESET "\n");
30
31 } else {
32 fprintf(stderr, "%s:%d:%d: error: %s\n",
33 buf_ptr(err->path),
34 err->line_start + 1, err->column_start + 1,
35 buf_ptr(err->msg));
36 }
37}
38
src/errmsg.hpp created+27
...@@ -0,0 +1,27 @@
1/*
2 * Copyright (c) 2015 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8#ifndef ZIG_ERRMSG_HPP
9#define ZIG_ERRMSG_HPP
10
11#include "buffer.hpp"
12#include "list.hpp"
13
14struct ErrorMsg {
15 int line_start;
16 int column_start;
17 int line_end;
18 int column_end;
19 Buf *msg;
20 Buf *path;
21 Buf *source;
22 ZigList<int> *line_offsets;
23};
24
25void print_err_msg(ErrorMsg *msg);
26
27#endif
src/os.cpp+4
...@@ -180,3 +180,7 @@ int os_get_cwd(Buf *out_cwd) {...@@ -180,3 +180,7 @@ int os_get_cwd(Buf *out_cwd) {
180180
181 return 0;181 return 0;
182}182}
183
184bool os_stderr_tty(void) {
185 return isatty(STDERR_FILENO);
186}
src/os.hpp+2
...@@ -29,4 +29,6 @@ int os_fetch_file_path(Buf *full_path, Buf *out_contents);...@@ -29,4 +29,6 @@ int os_fetch_file_path(Buf *full_path, Buf *out_contents);
29int os_get_cwd(Buf *out_cwd);29int os_get_cwd(Buf *out_cwd);
3030
3131
32bool os_stderr_tty(void);
33
32#endif34#endif
src/parser.cpp+65-53
...@@ -6,6 +6,8 @@...@@ -6,6 +6,8 @@
6 */6 */
77
8#include "parser.hpp"8#include "parser.hpp"
9#include "errmsg.hpp"
10#include "semantic_info.hpp"
911
10#include <stdarg.h>12#include <stdarg.h>
11#include <stdio.h>13#include <stdio.h>
...@@ -45,21 +47,6 @@ static const char *prefix_op_str(PrefixOp prefix_op) {...@@ -45,21 +47,6 @@ static const char *prefix_op_str(PrefixOp prefix_op) {
45 zig_unreachable();47 zig_unreachable();
46}48}
4749
48__attribute__ ((format (printf, 2, 3)))
49__attribute__ ((noreturn))
50static void ast_error(Token *token, const char *format, ...) {
51 int line = token->start_line + 1;
52 int column = token->start_column + 1;
53
54 va_list ap;
55 va_start(ap, format);
56 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
57 vfprintf(stderr, format, ap);
58 fprintf(stderr, "\n");
59 va_end(ap);
60 exit(EXIT_FAILURE);
61}
62
63const char *node_type_str(NodeType node_type) {50const char *node_type_str(NodeType node_type) {
64 switch (node_type) {51 switch (node_type) {
65 case NodeTypeRoot:52 case NodeTypeRoot:
...@@ -254,11 +241,35 @@ struct ParseContext {...@@ -254,11 +241,35 @@ struct ParseContext {
254 AstNode *root;241 AstNode *root;
255 ZigList<Token> *tokens;242 ZigList<Token> *tokens;
256 ZigList<AstNode *> *directive_list;243 ZigList<AstNode *> *directive_list;
244 ImportTableEntry *owner;
257};245};
258246
259static AstNode *ast_create_node_no_line_info(NodeType type) {247__attribute__ ((format (printf, 3, 4)))
248__attribute__ ((noreturn))
249static void ast_error(ParseContext *pc, Token *token, const char *format, ...) {
250 ErrorMsg *err = allocate<ErrorMsg>(1);
251 err->line_start = token->start_line;
252 err->column_start = token->start_column;
253 err->line_end = -1;
254 err->column_end = -1;
255
256 va_list ap;
257 va_start(ap, format);
258 err->msg = buf_vprintf(format, ap);
259 va_end(ap);
260
261 err->path = pc->owner->path;
262 err->source = pc->owner->source_code;
263 err->line_offsets = pc->owner->line_offsets;
264
265 print_err_msg(err);
266 exit(EXIT_FAILURE);
267}
268
269static AstNode *ast_create_node_no_line_info(ParseContext *pc, NodeType type) {
260 AstNode *node = allocate<AstNode>(1);270 AstNode *node = allocate<AstNode>(1);
261 node->type = type;271 node->type = type;
272 node->owner = pc->owner;
262 return node;273 return node;
263}274}
264275
...@@ -267,21 +278,21 @@ static void ast_update_node_line_info(AstNode *node, Token *first_token) {...@@ -267,21 +278,21 @@ static void ast_update_node_line_info(AstNode *node, Token *first_token) {
267 node->column = first_token->start_column;278 node->column = first_token->start_column;
268}279}
269280
270static AstNode *ast_create_node(NodeType type, Token *first_token) {281static AstNode *ast_create_node(ParseContext *pc, NodeType type, Token *first_token) {
271 AstNode *node = ast_create_node_no_line_info(type);282 AstNode *node = ast_create_node_no_line_info(pc, type);
272 ast_update_node_line_info(node, first_token);283 ast_update_node_line_info(node, first_token);
273 return node;284 return node;
274}285}
275286
276static AstNode *ast_create_node_with_node(NodeType type, AstNode *other_node) {287static AstNode *ast_create_node_with_node(ParseContext *pc, NodeType type, AstNode *other_node) {
277 AstNode *node = ast_create_node_no_line_info(type);288 AstNode *node = ast_create_node_no_line_info(pc, type);
278 node->line = other_node->line;289 node->line = other_node->line;
279 node->column = other_node->column;290 node->column = other_node->column;
280 return node;291 return node;
281}292}
282293
283static AstNode *ast_create_void_type_node(ParseContext *pc, Token *token) {294static AstNode *ast_create_void_type_node(ParseContext *pc, Token *token) {
284 AstNode *node = ast_create_node(NodeTypeType, token);295 AstNode *node = ast_create_node(pc, NodeTypeType, token);
285 node->data.type.type = AstNodeTypeTypePrimitive;296 node->data.type.type = AstNodeTypeTypePrimitive;
286 buf_init_from_str(&node->data.type.primitive_name, "void");297 buf_init_from_str(&node->data.type.primitive_name, "void");
287 return node;298 return node;
...@@ -331,7 +342,7 @@ __attribute__ ((noreturn))...@@ -331,7 +342,7 @@ __attribute__ ((noreturn))
331static void ast_invalid_token_error(ParseContext *pc, Token *token) {342static void ast_invalid_token_error(ParseContext *pc, Token *token) {
332 Buf token_value = BUF_INIT;343 Buf token_value = BUF_INIT;
333 ast_buf_from_token(pc, token, &token_value);344 ast_buf_from_token(pc, token, &token_value);
334 ast_error(token, "invalid token: '%s'", buf_ptr(&token_value));345 ast_error(pc, token, "invalid token: '%s'", buf_ptr(&token_value));
335}346}
336347
337static AstNode *ast_parse_expression(ParseContext *pc, int *token_index, bool mandatory);348static AstNode *ast_parse_expression(ParseContext *pc, int *token_index, bool mandatory);
...@@ -349,7 +360,7 @@ static AstNode *ast_parse_directive(ParseContext *pc, int token_index, int *new_...@@ -349,7 +360,7 @@ static AstNode *ast_parse_directive(ParseContext *pc, int token_index, int *new_
349 token_index += 1;360 token_index += 1;
350 ast_expect_token(pc, number_sign, TokenIdNumberSign);361 ast_expect_token(pc, number_sign, TokenIdNumberSign);
351362
352 AstNode *node = ast_create_node(NodeTypeDirective, number_sign);363 AstNode *node = ast_create_node(pc, NodeTypeDirective, number_sign);
353364
354 Token *name_symbol = &pc->tokens->at(token_index);365 Token *name_symbol = &pc->tokens->at(token_index);
355 token_index += 1;366 token_index += 1;
...@@ -399,7 +410,7 @@ static AstNode *ast_parse_type(ParseContext *pc, int token_index, int *new_token...@@ -399,7 +410,7 @@ static AstNode *ast_parse_type(ParseContext *pc, int token_index, int *new_token
399 Token *token = &pc->tokens->at(token_index);410 Token *token = &pc->tokens->at(token_index);
400 token_index += 1;411 token_index += 1;
401412
402 AstNode *node = ast_create_node(NodeTypeType, token);413 AstNode *node = ast_create_node(pc, NodeTypeType, token);
403414
404 if (token->id == TokenIdKeywordUnreachable) {415 if (token->id == TokenIdKeywordUnreachable) {
405 node->data.type.type = AstNodeTypeTypePrimitive;416 node->data.type.type = AstNodeTypeTypePrimitive;
...@@ -437,7 +448,7 @@ static AstNode *ast_parse_param_decl(ParseContext *pc, int token_index, int *new...@@ -437,7 +448,7 @@ static AstNode *ast_parse_param_decl(ParseContext *pc, int token_index, int *new
437 token_index += 1;448 token_index += 1;
438 ast_expect_token(pc, param_name, TokenIdSymbol);449 ast_expect_token(pc, param_name, TokenIdSymbol);
439450
440 AstNode *node = ast_create_node(NodeTypeParamDecl, param_name);451 AstNode *node = ast_create_node(pc, NodeTypeParamDecl, param_name);
441452
442453
443 ast_buf_from_token(pc, param_name, &node->data.param_decl.name);454 ast_buf_from_token(pc, param_name, &node->data.param_decl.name);
...@@ -544,21 +555,21 @@ static AstNode *ast_parse_primary_expr(ParseContext *pc, int *token_index, bool...@@ -544,21 +555,21 @@ static AstNode *ast_parse_primary_expr(ParseContext *pc, int *token_index, bool
544 Token *token = &pc->tokens->at(*token_index);555 Token *token = &pc->tokens->at(*token_index);
545556
546 if (token->id == TokenIdNumberLiteral) {557 if (token->id == TokenIdNumberLiteral) {
547 AstNode *node = ast_create_node(NodeTypeNumberLiteral, token);558 AstNode *node = ast_create_node(pc, NodeTypeNumberLiteral, token);
548 ast_buf_from_token(pc, token, &node->data.number);559 ast_buf_from_token(pc, token, &node->data.number);
549 *token_index += 1;560 *token_index += 1;
550 return node;561 return node;
551 } else if (token->id == TokenIdStringLiteral) {562 } else if (token->id == TokenIdStringLiteral) {
552 AstNode *node = ast_create_node(NodeTypeStringLiteral, token);563 AstNode *node = ast_create_node(pc, NodeTypeStringLiteral, token);
553 parse_string_literal(pc, token, &node->data.string);564 parse_string_literal(pc, token, &node->data.string);
554 *token_index += 1;565 *token_index += 1;
555 return node;566 return node;
556 } else if (token->id == TokenIdKeywordUnreachable) {567 } else if (token->id == TokenIdKeywordUnreachable) {
557 AstNode *node = ast_create_node(NodeTypeUnreachable, token);568 AstNode *node = ast_create_node(pc, NodeTypeUnreachable, token);
558 *token_index += 1;569 *token_index += 1;
559 return node;570 return node;
560 } else if (token->id == TokenIdSymbol) {571 } else if (token->id == TokenIdSymbol) {
561 AstNode *node = ast_create_node(NodeTypeSymbol, token);572 AstNode *node = ast_create_node(pc, NodeTypeSymbol, token);
562 ast_buf_from_token(pc, token, &node->data.symbol);573 ast_buf_from_token(pc, token, &node->data.symbol);
563 *token_index += 1;574 *token_index += 1;
564 return node;575 return node;
...@@ -592,7 +603,7 @@ static AstNode *ast_parse_fn_call_expr(ParseContext *pc, int *token_index, bool...@@ -592,7 +603,7 @@ static AstNode *ast_parse_fn_call_expr(ParseContext *pc, int *token_index, bool
592 if (l_paren->id != TokenIdLParen)603 if (l_paren->id != TokenIdLParen)
593 return primary_expr;604 return primary_expr;
594605
595 AstNode *node = ast_create_node_with_node(NodeTypeFnCallExpr, primary_expr);606 AstNode *node = ast_create_node_with_node(pc, NodeTypeFnCallExpr, primary_expr);
596 node->data.fn_call_expr.fn_ref_expr = primary_expr;607 node->data.fn_call_expr.fn_ref_expr = primary_expr;
597 ast_parse_fn_call_param_list(pc, *token_index, token_index, &node->data.fn_call_expr.params);608 ast_parse_fn_call_param_list(pc, *token_index, token_index, &node->data.fn_call_expr.params);
598609
...@@ -635,7 +646,7 @@ static AstNode *ast_parse_prefix_op_expr(ParseContext *pc, int *token_index, boo...@@ -635,7 +646,7 @@ static AstNode *ast_parse_prefix_op_expr(ParseContext *pc, int *token_index, boo
635 return ast_parse_fn_call_expr(pc, token_index, mandatory);646 return ast_parse_fn_call_expr(pc, token_index, mandatory);
636647
637 AstNode *primary_expr = ast_parse_fn_call_expr(pc, token_index, true);648 AstNode *primary_expr = ast_parse_fn_call_expr(pc, token_index, true);
638 AstNode *node = ast_create_node(NodeTypePrefixOpExpr, token);649 AstNode *node = ast_create_node(pc, NodeTypePrefixOpExpr, token);
639 node->data.prefix_op_expr.primary_expr = primary_expr;650 node->data.prefix_op_expr.primary_expr = primary_expr;
640 node->data.prefix_op_expr.prefix_op = prefix_op;651 node->data.prefix_op_expr.prefix_op = prefix_op;
641652
...@@ -656,7 +667,7 @@ static AstNode *ast_parse_cast_expression(ParseContext *pc, int *token_index, bo...@@ -656,7 +667,7 @@ static AstNode *ast_parse_cast_expression(ParseContext *pc, int *token_index, bo
656 return prefix_op_expr;667 return prefix_op_expr;
657 *token_index += 1;668 *token_index += 1;
658669
659 AstNode *node = ast_create_node(NodeTypeCastExpr, as_kw);670 AstNode *node = ast_create_node(pc, NodeTypeCastExpr, as_kw);
660 node->data.cast_expr.prefix_op_expr = prefix_op_expr;671 node->data.cast_expr.prefix_op_expr = prefix_op_expr;
661672
662 node->data.cast_expr.type = ast_parse_type(pc, *token_index, token_index);673 node->data.cast_expr.type = ast_parse_type(pc, *token_index, token_index);
...@@ -705,7 +716,7 @@ static AstNode *ast_parse_mult_expr(ParseContext *pc, int *token_index, bool man...@@ -705,7 +716,7 @@ static AstNode *ast_parse_mult_expr(ParseContext *pc, int *token_index, bool man
705716
706 AstNode *operand_2 = ast_parse_cast_expression(pc, token_index, true);717 AstNode *operand_2 = ast_parse_cast_expression(pc, token_index, true);
707718
708 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);719 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
709 node->data.bin_op_expr.op1 = operand_1;720 node->data.bin_op_expr.op1 = operand_1;
710 node->data.bin_op_expr.bin_op = mult_op;721 node->data.bin_op_expr.bin_op = mult_op;
711 node->data.bin_op_expr.op2 = operand_2;722 node->data.bin_op_expr.op2 = operand_2;
...@@ -753,7 +764,7 @@ static AstNode *ast_parse_add_expr(ParseContext *pc, int *token_index, bool mand...@@ -753,7 +764,7 @@ static AstNode *ast_parse_add_expr(ParseContext *pc, int *token_index, bool mand
753764
754 AstNode *operand_2 = ast_parse_mult_expr(pc, token_index, true);765 AstNode *operand_2 = ast_parse_mult_expr(pc, token_index, true);
755766
756 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);767 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
757 node->data.bin_op_expr.op1 = operand_1;768 node->data.bin_op_expr.op1 = operand_1;
758 node->data.bin_op_expr.bin_op = add_op;769 node->data.bin_op_expr.bin_op = add_op;
759 node->data.bin_op_expr.op2 = operand_2;770 node->data.bin_op_expr.op2 = operand_2;
...@@ -801,7 +812,7 @@ static AstNode *ast_parse_bit_shift_expr(ParseContext *pc, int *token_index, boo...@@ -801,7 +812,7 @@ static AstNode *ast_parse_bit_shift_expr(ParseContext *pc, int *token_index, boo
801812
802 AstNode *operand_2 = ast_parse_add_expr(pc, token_index, true);813 AstNode *operand_2 = ast_parse_add_expr(pc, token_index, true);
803814
804 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);815 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
805 node->data.bin_op_expr.op1 = operand_1;816 node->data.bin_op_expr.op1 = operand_1;
806 node->data.bin_op_expr.bin_op = bit_shift_op;817 node->data.bin_op_expr.bin_op = bit_shift_op;
807 node->data.bin_op_expr.op2 = operand_2;818 node->data.bin_op_expr.op2 = operand_2;
...@@ -825,7 +836,7 @@ static AstNode *ast_parse_bin_and_expr(ParseContext *pc, int *token_index, bool...@@ -825,7 +836,7 @@ static AstNode *ast_parse_bin_and_expr(ParseContext *pc, int *token_index, bool
825836
826 AstNode *operand_2 = ast_parse_bit_shift_expr(pc, token_index, true);837 AstNode *operand_2 = ast_parse_bit_shift_expr(pc, token_index, true);
827838
828 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);839 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
829 node->data.bin_op_expr.op1 = operand_1;840 node->data.bin_op_expr.op1 = operand_1;
830 node->data.bin_op_expr.bin_op = BinOpTypeBinAnd;841 node->data.bin_op_expr.bin_op = BinOpTypeBinAnd;
831 node->data.bin_op_expr.op2 = operand_2;842 node->data.bin_op_expr.op2 = operand_2;
...@@ -848,7 +859,7 @@ static AstNode *ast_parse_bin_xor_expr(ParseContext *pc, int *token_index, bool...@@ -848,7 +859,7 @@ static AstNode *ast_parse_bin_xor_expr(ParseContext *pc, int *token_index, bool
848859
849 AstNode *operand_2 = ast_parse_bin_and_expr(pc, token_index, true);860 AstNode *operand_2 = ast_parse_bin_and_expr(pc, token_index, true);
850861
851 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);862 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
852 node->data.bin_op_expr.op1 = operand_1;863 node->data.bin_op_expr.op1 = operand_1;
853 node->data.bin_op_expr.bin_op = BinOpTypeBinXor;864 node->data.bin_op_expr.bin_op = BinOpTypeBinXor;
854 node->data.bin_op_expr.op2 = operand_2;865 node->data.bin_op_expr.op2 = operand_2;
...@@ -871,7 +882,7 @@ static AstNode *ast_parse_bin_or_expr(ParseContext *pc, int *token_index, bool m...@@ -871,7 +882,7 @@ static AstNode *ast_parse_bin_or_expr(ParseContext *pc, int *token_index, bool m
871882
872 AstNode *operand_2 = ast_parse_bin_xor_expr(pc, token_index, true);883 AstNode *operand_2 = ast_parse_bin_xor_expr(pc, token_index, true);
873884
874 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);885 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
875 node->data.bin_op_expr.op1 = operand_1;886 node->data.bin_op_expr.op1 = operand_1;
876 node->data.bin_op_expr.bin_op = BinOpTypeBinOr;887 node->data.bin_op_expr.bin_op = BinOpTypeBinOr;
877 node->data.bin_op_expr.op2 = operand_2;888 node->data.bin_op_expr.op2 = operand_2;
...@@ -920,7 +931,7 @@ static AstNode *ast_parse_comparison_expr(ParseContext *pc, int *token_index, bo...@@ -920,7 +931,7 @@ static AstNode *ast_parse_comparison_expr(ParseContext *pc, int *token_index, bo
920931
921 AstNode *operand_2 = ast_parse_bin_or_expr(pc, token_index, true);932 AstNode *operand_2 = ast_parse_bin_or_expr(pc, token_index, true);
922933
923 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);934 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
924 node->data.bin_op_expr.op1 = operand_1;935 node->data.bin_op_expr.op1 = operand_1;
925 node->data.bin_op_expr.bin_op = cmp_op;936 node->data.bin_op_expr.bin_op = cmp_op;
926 node->data.bin_op_expr.op2 = operand_2;937 node->data.bin_op_expr.op2 = operand_2;
...@@ -943,7 +954,7 @@ static AstNode *ast_parse_bool_and_expr(ParseContext *pc, int *token_index, bool...@@ -943,7 +954,7 @@ static AstNode *ast_parse_bool_and_expr(ParseContext *pc, int *token_index, bool
943954
944 AstNode *operand_2 = ast_parse_comparison_expr(pc, token_index, true);955 AstNode *operand_2 = ast_parse_comparison_expr(pc, token_index, true);
945956
946 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);957 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
947 node->data.bin_op_expr.op1 = operand_1;958 node->data.bin_op_expr.op1 = operand_1;
948 node->data.bin_op_expr.bin_op = BinOpTypeBoolAnd;959 node->data.bin_op_expr.bin_op = BinOpTypeBoolAnd;
949 node->data.bin_op_expr.op2 = operand_2;960 node->data.bin_op_expr.op2 = operand_2;
...@@ -958,7 +969,7 @@ static AstNode *ast_parse_return_expr(ParseContext *pc, int *token_index, bool m...@@ -958,7 +969,7 @@ static AstNode *ast_parse_return_expr(ParseContext *pc, int *token_index, bool m
958 Token *return_tok = &pc->tokens->at(*token_index);969 Token *return_tok = &pc->tokens->at(*token_index);
959 if (return_tok->id == TokenIdKeywordReturn) {970 if (return_tok->id == TokenIdKeywordReturn) {
960 *token_index += 1;971 *token_index += 1;
961 AstNode *node = ast_create_node(NodeTypeReturnExpr, return_tok);972 AstNode *node = ast_create_node(pc, NodeTypeReturnExpr, return_tok);
962 node->data.return_expr.expr = ast_parse_expression(pc, token_index, false);973 node->data.return_expr.expr = ast_parse_expression(pc, token_index, false);
963 return node;974 return node;
964 } else if (mandatory) {975 } else if (mandatory) {
...@@ -983,7 +994,7 @@ static AstNode *ast_parse_bool_or_expr(ParseContext *pc, int *token_index, bool...@@ -983,7 +994,7 @@ static AstNode *ast_parse_bool_or_expr(ParseContext *pc, int *token_index, bool
983994
984 AstNode *operand_2 = ast_parse_bool_and_expr(pc, token_index, true);995 AstNode *operand_2 = ast_parse_bool_and_expr(pc, token_index, true);
985996
986 AstNode *node = ast_create_node(NodeTypeBinOpExpr, token);997 AstNode *node = ast_create_node(pc, NodeTypeBinOpExpr, token);
987 node->data.bin_op_expr.op1 = operand_1;998 node->data.bin_op_expr.op1 = operand_1;
988 node->data.bin_op_expr.bin_op = BinOpTypeBoolOr;999 node->data.bin_op_expr.bin_op = BinOpTypeBoolOr;
989 node->data.bin_op_expr.op2 = operand_2;1000 node->data.bin_op_expr.op2 = operand_2;
...@@ -1046,7 +1057,7 @@ static AstNode *ast_parse_block(ParseContext *pc, int *token_index, bool mandato...@@ -1046,7 +1057,7 @@ static AstNode *ast_parse_block(ParseContext *pc, int *token_index, bool mandato
1046 }1057 }
1047 *token_index += 1;1058 *token_index += 1;
10481059
1049 AstNode *node = ast_create_node(NodeTypeBlock, l_brace);1060 AstNode *node = ast_create_node(pc, NodeTypeBlock, l_brace);
10501061
1051 for (;;) {1062 for (;;) {
1052 Token *token = &pc->tokens->at(*token_index);1063 Token *token = &pc->tokens->at(*token_index);
...@@ -1092,7 +1103,7 @@ static AstNode *ast_parse_fn_proto(ParseContext *pc, int *token_index, bool mand...@@ -1092,7 +1103,7 @@ static AstNode *ast_parse_fn_proto(ParseContext *pc, int *token_index, bool mand
1092 return nullptr;1103 return nullptr;
1093 }1104 }
10941105
1095 AstNode *node = ast_create_node(NodeTypeFnProto, token);1106 AstNode *node = ast_create_node(pc, NodeTypeFnProto, token);
1096 node->data.fn_proto.visib_mod = visib_mod;1107 node->data.fn_proto.visib_mod = visib_mod;
1097 node->data.fn_proto.directives = pc->directive_list;1108 node->data.fn_proto.directives = pc->directive_list;
1098 pc->directive_list = nullptr;1109 pc->directive_list = nullptr;
...@@ -1125,7 +1136,7 @@ static AstNode *ast_parse_fn_def(ParseContext *pc, int *token_index, bool mandat...@@ -1125,7 +1136,7 @@ static AstNode *ast_parse_fn_def(ParseContext *pc, int *token_index, bool mandat
1125 AstNode *fn_proto = ast_parse_fn_proto(pc, token_index, mandatory);1136 AstNode *fn_proto = ast_parse_fn_proto(pc, token_index, mandatory);
1126 if (!fn_proto)1137 if (!fn_proto)
1127 return nullptr;1138 return nullptr;
1128 AstNode *node = ast_create_node_with_node(NodeTypeFnDef, fn_proto);1139 AstNode *node = ast_create_node_with_node(pc, NodeTypeFnDef, fn_proto);
11291140
1130 node->data.fn_def.fn_proto = fn_proto;1141 node->data.fn_def.fn_proto = fn_proto;
1131 node->data.fn_def.body = ast_parse_block(pc, token_index, true);1142 node->data.fn_def.body = ast_parse_block(pc, token_index, true);
...@@ -1138,7 +1149,7 @@ FnDecl : FnProto token(Semicolon)...@@ -1138,7 +1149,7 @@ FnDecl : FnProto token(Semicolon)
1138*/1149*/
1139static AstNode *ast_parse_fn_decl(ParseContext *pc, int token_index, int *new_token_index) {1150static AstNode *ast_parse_fn_decl(ParseContext *pc, int token_index, int *new_token_index) {
1140 AstNode *fn_proto = ast_parse_fn_proto(pc, &token_index, true);1151 AstNode *fn_proto = ast_parse_fn_proto(pc, &token_index, true);
1141 AstNode *node = ast_create_node_with_node(NodeTypeFnDecl, fn_proto);1152 AstNode *node = ast_create_node_with_node(pc, NodeTypeFnDecl, fn_proto);
11421153
1143 node->data.fn_decl.fn_proto = fn_proto;1154 node->data.fn_decl.fn_proto = fn_proto;
11441155
...@@ -1166,7 +1177,7 @@ static AstNode *ast_parse_extern_block(ParseContext *pc, int *token_index, bool...@@ -1166,7 +1177,7 @@ static AstNode *ast_parse_extern_block(ParseContext *pc, int *token_index, bool
1166 }1177 }
1167 *token_index += 1;1178 *token_index += 1;
11681179
1169 AstNode *node = ast_create_node(NodeTypeExternBlock, extern_kw);1180 AstNode *node = ast_create_node(pc, NodeTypeExternBlock, extern_kw);
11701181
1171 node->data.extern_block.directives = pc->directive_list;1182 node->data.extern_block.directives = pc->directive_list;
1172 pc->directive_list = nullptr;1183 pc->directive_list = nullptr;
...@@ -1184,7 +1195,7 @@ static AstNode *ast_parse_extern_block(ParseContext *pc, int *token_index, bool...@@ -1184,7 +1195,7 @@ static AstNode *ast_parse_extern_block(ParseContext *pc, int *token_index, bool
1184 Token *token = &pc->tokens->at(*token_index);1195 Token *token = &pc->tokens->at(*token_index);
1185 if (token->id == TokenIdRBrace) {1196 if (token->id == TokenIdRBrace) {
1186 if (pc->directive_list->length > 0) {1197 if (pc->directive_list->length > 0) {
1187 ast_error(directive_token, "invalid directive");1198 ast_error(pc, directive_token, "invalid directive");
1188 }1199 }
1189 pc->directive_list = nullptr;1200 pc->directive_list = nullptr;
11901201
...@@ -1216,7 +1227,7 @@ static AstNode *ast_parse_root_export_decl(ParseContext *pc, int *token_index, b...@@ -1216,7 +1227,7 @@ static AstNode *ast_parse_root_export_decl(ParseContext *pc, int *token_index, b
12161227
1217 *token_index += 2;1228 *token_index += 2;
12181229
1219 AstNode *node = ast_create_node(NodeTypeRootExportDecl, export_kw);1230 AstNode *node = ast_create_node(pc, NodeTypeRootExportDecl, export_kw);
1220 node->data.root_export_decl.directives = pc->directive_list;1231 node->data.root_export_decl.directives = pc->directive_list;
1221 pc->directive_list = nullptr;1232 pc->directive_list = nullptr;
12221233
...@@ -1254,7 +1265,7 @@ static AstNode *ast_parse_use(ParseContext *pc, int *token_index, bool mandatory...@@ -1254,7 +1265,7 @@ static AstNode *ast_parse_use(ParseContext *pc, int *token_index, bool mandatory
1254 *token_index += 1;1265 *token_index += 1;
1255 ast_expect_token(pc, semicolon, TokenIdSemicolon);1266 ast_expect_token(pc, semicolon, TokenIdSemicolon);
12561267
1257 AstNode *node = ast_create_node(NodeTypeUse, use_kw);1268 AstNode *node = ast_create_node(pc, NodeTypeUse, use_kw);
12581269
1259 parse_string_literal(pc, use_name, &node->data.use.path);1270 parse_string_literal(pc, use_name, &node->data.use.path);
12601271
...@@ -1299,7 +1310,7 @@ static void ast_parse_top_level_decls(ParseContext *pc, int *token_index, ZigLis...@@ -1299,7 +1310,7 @@ static void ast_parse_top_level_decls(ParseContext *pc, int *token_index, ZigLis
1299 }1310 }
13001311
1301 if (pc->directive_list->length > 0) {1312 if (pc->directive_list->length > 0) {
1302 ast_error(directive_token, "invalid directive");1313 ast_error(pc, directive_token, "invalid directive");
1303 }1314 }
1304 pc->directive_list = nullptr;1315 pc->directive_list = nullptr;
13051316
...@@ -1312,7 +1323,7 @@ static void ast_parse_top_level_decls(ParseContext *pc, int *token_index, ZigLis...@@ -1312,7 +1323,7 @@ static void ast_parse_top_level_decls(ParseContext *pc, int *token_index, ZigLis
1312Root : many(TopLevelDecl) token(EOF)1323Root : many(TopLevelDecl) token(EOF)
1313 */1324 */
1314static AstNode *ast_parse_root(ParseContext *pc, int *token_index) {1325static AstNode *ast_parse_root(ParseContext *pc, int *token_index) {
1315 AstNode *node = ast_create_node(NodeTypeRoot, &pc->tokens->at(*token_index));1326 AstNode *node = ast_create_node(pc, NodeTypeRoot, &pc->tokens->at(*token_index));
13161327
1317 ast_parse_top_level_decls(pc, token_index, &node->data.root.top_level_decls);1328 ast_parse_top_level_decls(pc, token_index, &node->data.root.top_level_decls);
13181329
...@@ -1323,8 +1334,9 @@ static AstNode *ast_parse_root(ParseContext *pc, int *token_index) {...@@ -1323,8 +1334,9 @@ static AstNode *ast_parse_root(ParseContext *pc, int *token_index) {
1323 return node;1334 return node;
1324}1335}
13251336
1326AstNode *ast_parse(Buf *buf, ZigList<Token> *tokens) {1337AstNode *ast_parse(Buf *buf, ZigList<Token> *tokens, ImportTableEntry *owner) {
1327 ParseContext pc = {0};1338 ParseContext pc = {0};
1339 pc.owner = owner;
1328 pc.buf = buf;1340 pc.buf = buf;
1329 pc.tokens = tokens;1341 pc.tokens = tokens;
1330 int token_index = 0;1342 int token_index = 0;
src/parser.hpp+3-2
...@@ -14,6 +14,7 @@...@@ -14,6 +14,7 @@
1414
15struct AstNode;15struct AstNode;
16struct CodeGenNode;16struct CodeGenNode;
17struct ImportTableEntry;
1718
18enum NodeType {19enum NodeType {
19 NodeTypeRoot,20 NodeTypeRoot,
...@@ -166,10 +167,10 @@ struct AstNodeUse {...@@ -166,10 +167,10 @@ struct AstNodeUse {
166167
167struct AstNode {168struct AstNode {
168 enum NodeType type;169 enum NodeType type;
169 AstNode *parent;
170 int line;170 int line;
171 int column;171 int column;
172 CodeGenNode *codegen_node;172 CodeGenNode *codegen_node;
173 ImportTableEntry *owner;
173 union {174 union {
174 AstNodeRoot root;175 AstNodeRoot root;
175 AstNodeRootExportDecl root_export_decl;176 AstNodeRootExportDecl root_export_decl;
...@@ -198,7 +199,7 @@ void ast_token_error(Token *token, const char *format, ...);...@@ -198,7 +199,7 @@ void ast_token_error(Token *token, const char *format, ...);
198199
199200
200// This function is provided by generated code, generated by parsergen.cpp201// This function is provided by generated code, generated by parsergen.cpp
201AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens);202AstNode * ast_parse(Buf *buf, ZigList<Token> *tokens, ImportTableEntry *owner);
202203
203const char *node_type_str(NodeType node_type);204const char *node_type_str(NodeType node_type);
204205
src/semantic_info.hpp+4-1
...@@ -11,6 +11,7 @@...@@ -11,6 +11,7 @@
11#include "codegen.hpp"11#include "codegen.hpp"
12#include "hash_map.hpp"12#include "hash_map.hpp"
13#include "zig_llvm.hpp"13#include "zig_llvm.hpp"
14#include "errmsg.hpp"
1415
15struct FnTableEntry;16struct FnTableEntry;
1617
...@@ -30,6 +31,8 @@ struct ImportTableEntry {...@@ -30,6 +31,8 @@ struct ImportTableEntry {
30 AstNode *root;31 AstNode *root;
31 Buf *path; // relative to root_source_dir32 Buf *path; // relative to root_source_dir
32 LLVMZigDIFile *di_file;33 LLVMZigDIFile *di_file;
34 Buf *source_code;
35 ZigList<int> *line_offsets;
3336
34 // reminder: hash tables must be initialized before use37 // reminder: hash tables must be initialized before use
35 HashMap<Buf *, FnTableEntry *, buf_hash, buf_eql_buf> fn_table;38 HashMap<Buf *, FnTableEntry *, buf_hash, buf_eql_buf> fn_table;
...@@ -47,7 +50,7 @@ struct FnTableEntry {...@@ -47,7 +50,7 @@ struct FnTableEntry {
4750
48struct CodeGen {51struct CodeGen {
49 LLVMModuleRef module;52 LLVMModuleRef module;
50 ZigList<ErrorMsg> errors;53 ZigList<ErrorMsg*> errors;
51 LLVMBuilderRef builder;54 LLVMBuilderRef builder;
52 LLVMZigDIBuilder *dbuilder;55 LLVMZigDIBuilder *dbuilder;
53 LLVMZigDICompileUnit *compile_unit;56 LLVMZigDICompileUnit *compile_unit;
src/tokenizer.cpp+26-17
...@@ -104,6 +104,7 @@ enum TokenizeState {...@@ -104,6 +104,7 @@ enum TokenizeState {
104 TokenizeStateBang,104 TokenizeStateBang,
105 TokenizeStateLessThan,105 TokenizeStateLessThan,
106 TokenizeStateGreaterThan,106 TokenizeStateGreaterThan,
107 TokenizeStateError,
107};108};
108109
109110
...@@ -116,27 +117,25 @@ struct Tokenize {...@@ -116,27 +117,25 @@ struct Tokenize {
116 int column;117 int column;
117 Token *cur_tok;118 Token *cur_tok;
118 int multi_line_comment_count;119 int multi_line_comment_count;
120 Tokenization *out;
119};121};
120122
121__attribute__ ((format (printf, 2, 3)))123__attribute__ ((format (printf, 2, 3)))
122static void tokenize_error(Tokenize *t, const char *format, ...) {124static void tokenize_error(Tokenize *t, const char *format, ...) {
123 int line;125 t->state = TokenizeStateError;
124 int column;126
125 if (t->cur_tok) {127 if (t->cur_tok) {
126 line = t->cur_tok->start_line + 1;128 t->out->err_line = t->cur_tok->start_line;
127 column = t->cur_tok->start_column + 1;129 t->out->err_column = t->cur_tok->start_column;
128 } else {130 } else {
129 line = t->line + 1;131 t->out->err_line = t->line;
130 column = t->column + 1;132 t->out->err_column = t->column;
131 }133 }
132134
133 va_list ap;135 va_list ap;
134 va_start(ap, format);136 va_start(ap, format);
135 fprintf(stderr, "Error: Line %d, column %d: ", line, column);137 t->out->err = buf_vprintf(format, ap);
136 vfprintf(stderr, format, ap);
137 fprintf(stderr, "\n");
138 va_end(ap);138 va_end(ap);
139 exit(EXIT_FAILURE);
140}139}
141140
142static void begin_token(Tokenize *t, TokenId id) {141static void begin_token(Tokenize *t, TokenId id) {
...@@ -187,13 +186,20 @@ static void end_token(Tokenize *t) {...@@ -187,13 +186,20 @@ static void end_token(Tokenize *t) {
187 t->cur_tok = nullptr;186 t->cur_tok = nullptr;
188}187}
189188
190ZigList<Token> *tokenize(Buf *buf) {189void tokenize(Buf *buf, Tokenization *out) {
191 Tokenize t = {0};190 Tokenize t = {0};
192 t.tokens = allocate<ZigList<Token>>(1);191 t.out = out;
192 t.tokens = out->tokens = allocate<ZigList<Token>>(1);
193 t.buf = buf;193 t.buf = buf;
194
195 out->line_offsets = allocate<ZigList<int>>(1);
196
197 out->line_offsets->append(0);
194 for (t.pos = 0; t.pos < buf_len(t.buf); t.pos += 1) {198 for (t.pos = 0; t.pos < buf_len(t.buf); t.pos += 1) {
195 uint8_t c = buf_ptr(t.buf)[t.pos];199 uint8_t c = buf_ptr(t.buf)[t.pos];
196 switch (t.state) {200 switch (t.state) {
201 case TokenizeStateError:
202 break;
197 case TokenizeStateStart:203 case TokenizeStateStart:
198 switch (c) {204 switch (c) {
199 case WHITESPACE:205 case WHITESPACE:
...@@ -509,6 +515,7 @@ ZigList<Token> *tokenize(Buf *buf) {...@@ -509,6 +515,7 @@ ZigList<Token> *tokenize(Buf *buf) {
509 break;515 break;
510 }516 }
511 if (c == '\n') {517 if (c == '\n') {
518 out->line_offsets->append(t.pos + 1);
512 t.line += 1;519 t.line += 1;
513 t.column = 0;520 t.column = 0;
514 } else {521 } else {
...@@ -518,6 +525,7 @@ ZigList<Token> *tokenize(Buf *buf) {...@@ -518,6 +525,7 @@ ZigList<Token> *tokenize(Buf *buf) {
518 // EOF525 // EOF
519 switch (t.state) {526 switch (t.state) {
520 case TokenizeStateStart:527 case TokenizeStateStart:
528 case TokenizeStateError:
521 break;529 break;
522 case TokenizeStateString:530 case TokenizeStateString:
523 tokenize_error(&t, "unterminated string");531 tokenize_error(&t, "unterminated string");
...@@ -544,11 +552,12 @@ ZigList<Token> *tokenize(Buf *buf) {...@@ -544,11 +552,12 @@ ZigList<Token> *tokenize(Buf *buf) {
544 tokenize_error(&t, "unterminated multi-line comment");552 tokenize_error(&t, "unterminated multi-line comment");
545 break;553 break;
546 }554 }
547 t.pos = -1;555 if (t.state != TokenizeStateError) {
548 begin_token(&t, TokenIdEof);556 t.pos = -1;
549 end_token(&t);557 begin_token(&t, TokenIdEof);
550 assert(!t.cur_tok);558 end_token(&t);
551 return t.tokens;559 assert(!t.cur_tok);
560 }
552}561}
553562
554static const char * token_name(Token *token) {563static const char * token_name(Token *token) {
src/tokenizer.hpp+11-1
...@@ -65,7 +65,17 @@ struct Token {...@@ -65,7 +65,17 @@ struct Token {
65 int start_column;65 int start_column;
66};66};
6767
68ZigList<Token> *tokenize(Buf *buf);68struct Tokenization {
69 ZigList<Token> *tokens;
70 ZigList<int> *line_offsets;
71
72 // if an error occurred
73 Buf *err;
74 int err_line;
75 int err_column;
76};
77
78void tokenize(Buf *buf, Tokenization *out_tokenization);
6979
70void print_tokens(Buf *buf, ZigList<Token> *tokens);80void print_tokens(Buf *buf, ZigList<Token> *tokens);
7181
test/run_tests.cpp+11-11
...@@ -139,7 +139,7 @@ static void add_compile_failure_test_cases(void) {...@@ -139,7 +139,7 @@ static void add_compile_failure_test_cases(void) {
139 add_compile_fail_case("multiple function definitions", R"SOURCE(139 add_compile_fail_case("multiple function definitions", R"SOURCE(
140fn a() {}140fn a() {}
141fn a() {}141fn a() {}
142 )SOURCE", 1, "Line 3, column 1: redefinition of 'a'");142 )SOURCE", 1, ".tmp_source.zig:3:1: error: redefinition of 'a'");
143143
144 add_compile_fail_case("bad directive", R"SOURCE(144 add_compile_fail_case("bad directive", R"SOURCE(
145#bogus1("")145#bogus1("")
...@@ -148,37 +148,37 @@ extern {...@@ -148,37 +148,37 @@ extern {
148}148}
149#bogus2("")149#bogus2("")
150fn a() {}150fn a() {}
151 )SOURCE", 2, "Line 2, column 1: invalid directive: 'bogus1'",151 )SOURCE", 2, ".tmp_source.zig:2:1: error: invalid directive: 'bogus1'",
152 "Line 6, column 1: invalid directive: 'bogus2'");152 ".tmp_source.zig:6:1: error: invalid directive: 'bogus2'");
153153
154 add_compile_fail_case("unreachable with return", R"SOURCE(154 add_compile_fail_case("unreachable with return", R"SOURCE(
155fn a() -> unreachable {return;}155fn a() -> unreachable {return;}
156 )SOURCE", 1, "Line 2, column 24: return statement in function with unreachable return type");156 )SOURCE", 1, ".tmp_source.zig:2:24: error: return statement in function with unreachable return type");
157157
158 add_compile_fail_case("control reaches end of non-void function", R"SOURCE(158 add_compile_fail_case("control reaches end of non-void function", R"SOURCE(
159fn a() -> i32 {}159fn a() -> i32 {}
160 )SOURCE", 1, "Line 2, column 1: control reaches end of non-void function");160 )SOURCE", 1, ".tmp_source.zig:2:1: error: control reaches end of non-void function");
161161
162 add_compile_fail_case("undefined function call", R"SOURCE(162 add_compile_fail_case("undefined function call", R"SOURCE(
163fn a() {163fn a() {
164 b();164 b();
165}165}
166 )SOURCE", 1, "Line 3, column 5: undefined function: 'b'");166 )SOURCE", 1, ".tmp_source.zig:3:5: error: undefined function: 'b'");
167167
168 add_compile_fail_case("wrong number of arguments", R"SOURCE(168 add_compile_fail_case("wrong number of arguments", R"SOURCE(
169fn a() {169fn a() {
170 b(1);170 b(1);
171}171}
172fn b(a: i32, b: i32, c: i32) { }172fn b(a: i32, b: i32, c: i32) { }
173 )SOURCE", 1, "Line 3, column 5: wrong number of arguments. Expected 3, got 1.");173 )SOURCE", 1, ".tmp_source.zig:3:5: error: wrong number of arguments. Expected 3, got 1.");
174174
175 add_compile_fail_case("invalid type", R"SOURCE(175 add_compile_fail_case("invalid type", R"SOURCE(
176fn a() -> bogus {}176fn a() -> bogus {}
177 )SOURCE", 1, "Line 2, column 11: invalid type name: 'bogus'");177 )SOURCE", 1, ".tmp_source.zig:2:11: error: invalid type name: 'bogus'");
178178
179 add_compile_fail_case("pointer to unreachable", R"SOURCE(179 add_compile_fail_case("pointer to unreachable", R"SOURCE(
180fn a() -> *mut unreachable {}180fn a() -> *mut unreachable {}
181 )SOURCE", 1, "Line 2, column 11: pointer to unreachable not allowed");181 )SOURCE", 1, ".tmp_source.zig:2:11: error: pointer to unreachable not allowed");
182182
183 add_compile_fail_case("unreachable code", R"SOURCE(183 add_compile_fail_case("unreachable code", R"SOURCE(
184fn a() {184fn a() {
...@@ -187,12 +187,12 @@ fn a() {...@@ -187,12 +187,12 @@ fn a() {
187}187}
188188
189fn b() {}189fn b() {}
190 )SOURCE", 1, "Line 4, column 5: unreachable code");190 )SOURCE", 1, ".tmp_source.zig:4:5: error: unreachable code");
191191
192 add_compile_fail_case("bad version string", R"SOURCE(192 add_compile_fail_case("bad version string", R"SOURCE(
193#version("aoeu")193#version("aoeu")
194export executable "test";194export executable "test";
195 )SOURCE", 1, "Line 2, column 1: invalid version string");195 )SOURCE", 1, ".tmp_source.zig:2:1: error: invalid version string");
196}196}
197197
198static void print_compiler_invokation(TestCase *test_case, Buf *zig_stderr) {198static void print_compiler_invokation(TestCase *test_case, Buf *zig_stderr) {