authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-05 20:26:58-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-05 20:26:58-07:00
loge71521335ac656f94892ee049cc4814ecb772b30
treebe4c24abec3ac3e434f70b6ba754e7d681dd8bd1
parente09932928ac16681d71127f3d5cc0c488f2fdcc5

preprocessor detects #include directives

3 files changed, 264 insertions(+), 44 deletions(-)

src/buffer.hpp created+82
...@@ -0,0 +1,82 @@
1/*
2 * Copyright (c) 2015 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8#ifndef ZIG_BUFFER_HPP
9#define ZIG_BUFFER_HPP
10
11#include "list.hpp"
12
13#include <assert.h>
14
15struct Buf {
16 ZigList<char> list;
17};
18
19static inline int buf_len(Buf *buf) {
20 return buf->list.length - 1;
21}
22
23static inline char *buf_ptr(Buf *buf) {
24 return buf->list.items;
25}
26
27static inline void buf_resize(Buf *buf, int new_len) {
28 buf->list.resize(new_len + 1);
29 buf->list.at(buf_len(buf)) = 0;
30}
31
32static inline Buf *buf_alloc(void) {
33 Buf *buf = allocate<Buf>(1);
34 buf_resize(buf, 0);
35 return buf;
36}
37
38static inline Buf *buf_alloc_fixed(int size) {
39 Buf *buf = allocate<Buf>(1);
40 buf_resize(buf, size);
41 return buf;
42}
43
44static inline void buf_deinit(Buf *buf) {
45 buf->list.deinit();
46}
47
48static inline Buf *buf_from_mem(char *ptr, int len) {
49 Buf *buf = allocate<Buf>(1);
50 buf->list.resize(len + 1);
51 memcpy(buf_ptr(buf), ptr, len);
52 buf->list.at(buf_len(buf)) = 0;
53 return buf;
54}
55
56static inline Buf *buf_slice(Buf *in_buf, int start, int end) {
57 assert(start >= 0);
58 assert(end >= 0);
59 assert(start < buf_len(in_buf));
60 assert(end <= buf_len(in_buf));
61 Buf *out_buf = allocate<Buf>(1);
62 out_buf->list.resize(end - start + 1);
63 memcpy(buf_ptr(out_buf), buf_ptr(in_buf) + start, end - start);
64 out_buf->list.at(buf_len(out_buf)) = 0;
65 return out_buf;
66}
67
68static inline void buf_append_str(Buf *buf, const char *str, int str_len) {
69 if (str_len == -1)
70 str_len = strlen(str);
71
72 int old_len = buf_len(buf);
73 buf_resize(buf, old_len + str_len);
74 memcpy(buf_ptr(buf) + old_len, str, str_len);
75 buf->list.at(buf_len(buf)) = 0;
76}
77
78static inline void buf_append_buf(Buf *buf, Buf *append_buf) {
79 buf_append_str(buf, buf_ptr(append_buf), buf_len(append_buf));
80}
81
82#endif
src/list.hpp+1-1
...@@ -15,7 +15,7 @@...@@ -15,7 +15,7 @@
15template<typename T>15template<typename T>
16struct ZigList {16struct ZigList {
17 void deinit() {17 void deinit() {
18 deallocate(items);18 free(items);
19 }19 }
20 void append(T item) {20 void append(T item) {
21 ensure_capacity(length + 1);21 ensure_capacity(length + 1);
src/main.cpp+181-43
...@@ -8,6 +8,8 @@...@@ -8,6 +8,8 @@
8#include "config.h"8#include "config.h"
9#include "util.hpp"9#include "util.hpp"
10#include "list.hpp"10#include "list.hpp"
11#include "buffer.hpp"
12
11#include <stdio.h>13#include <stdio.h>
12#include <string.h>14#include <string.h>
13#include <stdlib.h>15#include <stdlib.h>
...@@ -19,25 +21,6 @@...@@ -19,25 +21,6 @@
19#include <sys/stat.h>21#include <sys/stat.h>
20#include <unistd.h>22#include <unistd.h>
2123
22struct Buf {
23 int len;
24 char ptr[0];
25};
26
27static Buf *alloc_buf(int size) {
28 Buf *buf = (Buf *)allocate_nonzero<char>(sizeof(Buf) + size + 1);
29 buf->len = size;
30 buf->ptr[buf->len] = 0;
31 return buf;
32}
33
34/*
35static void fprint_buf(FILE *f, Buf *buf) {
36 if (fwrite(buf->ptr, 1, buf->len, f))
37 zig_panic("error writing: %s", strerror(errno));
38}
39*/
40
41static int usage(char *arg0) {24static int usage(char *arg0) {
42 fprintf(stderr, "Usage: %s --output outfile code.zig\n"25 fprintf(stderr, "Usage: %s --output outfile code.zig\n"
43 "Other options:\n"26 "Other options:\n"
...@@ -46,7 +29,7 @@ static int usage(char *arg0) {...@@ -46,7 +29,7 @@ static int usage(char *arg0) {
46 return EXIT_FAILURE;29 return EXIT_FAILURE;
47}30}
4831
49static struct Buf *fetch_file(FILE *f) {32static Buf *fetch_file(FILE *f) {
50 int fd = fileno(f);33 int fd = fileno(f);
51 struct stat st;34 struct stat st;
52 if (fstat(fd, &st))35 if (fstat(fd, &st))
...@@ -56,9 +39,9 @@ static struct Buf *fetch_file(FILE *f) {...@@ -56,9 +39,9 @@ static struct Buf *fetch_file(FILE *f) {
56 zig_panic("file too big");39 zig_panic("file too big");
57 int size = (int)big_size;40 int size = (int)big_size;
5841
59 Buf *buf = alloc_buf(size);42 Buf *buf = buf_alloc_fixed(size);
60 size_t amt_read = fread(buf->ptr, 1, buf->len, f);43 size_t amt_read = fread(buf_ptr(buf), 1, buf_len(buf), f);
61 if (amt_read != (size_t)buf->len)44 if (amt_read != (size_t)buf_len(buf))
62 zig_panic("error reading: %s", strerror(errno));45 zig_panic("error reading: %s", strerror(errno));
6346
64 return buf;47 return buf;
...@@ -138,6 +121,12 @@ static struct Buf *fetch_file(FILE *f) {...@@ -138,6 +121,12 @@ static struct Buf *fetch_file(FILE *f) {
138 case 'Y': \121 case 'Y': \
139 case 'Z'122 case 'Z'
140123
124#define SYMBOL_CHAR \
125 ALPHA: \
126 case DIGIT: \
127 case '_'
128
129
141enum TokenId {130enum TokenId {
142 TokenIdDirective,131 TokenIdDirective,
143 TokenIdSymbol,132 TokenIdSymbol,
...@@ -157,6 +146,8 @@ struct Token {...@@ -157,6 +146,8 @@ struct Token {
157 TokenId id;146 TokenId id;
158 int start_pos;147 int start_pos;
159 int end_pos;148 int end_pos;
149 int start_line;
150 int start_column;
160};151};
161152
162enum TokenizeState {153enum TokenizeState {
...@@ -178,10 +169,21 @@ struct Tokenize {...@@ -178,10 +169,21 @@ struct Tokenize {
178169
179__attribute__ ((format (printf, 2, 3)))170__attribute__ ((format (printf, 2, 3)))
180static void tokenize_error(Tokenize *t, const char *format, ...) {171static void tokenize_error(Tokenize *t, const char *format, ...) {
172 int line;
173 int column;
174 if (t->cur_tok) {
175 line = t->cur_tok->start_line + 1;
176 column = t->cur_tok->start_column + 1;
177 } else {
178 line = t->line + 1;
179 column = t->column + 1;
180 }
181
181 va_list ap;182 va_list ap;
182 va_start(ap, format);183 va_start(ap, format);
183 fprintf(stderr, "Error. Line %d, column %d: ", t->line + 1, t->column + 1);184 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
184 vfprintf(stderr, format, ap);185 vfprintf(stderr, format, ap);
186 fprintf(stderr, "\n");
185 va_end(ap);187 va_end(ap);
186 exit(EXIT_FAILURE);188 exit(EXIT_FAILURE);
187}189}
...@@ -190,6 +192,8 @@ static void begin_token(Tokenize *t, TokenId id) {...@@ -190,6 +192,8 @@ static void begin_token(Tokenize *t, TokenId id) {
190 assert(!t->cur_tok);192 assert(!t->cur_tok);
191 t->tokens->add_one();193 t->tokens->add_one();
192 Token *token = &t->tokens->last();194 Token *token = &t->tokens->last();
195 token->start_line = t->line;
196 token->start_column = t->column;
193 token->id = id;197 token->id = id;
194 token->start_pos = t->pos;198 token->start_pos = t->pos;
195 t->cur_tok = token;199 t->cur_tok = token;
...@@ -205,11 +209,24 @@ static void put_back(Tokenize *t, int count) {...@@ -205,11 +209,24 @@ static void put_back(Tokenize *t, int count) {
205 t->pos -= count;209 t->pos -= count;
206}210}
207211
212static void end_directive(Tokenize *t) {
213 assert(t->cur_tok);
214 t->cur_tok->end_pos = t->pos;
215 t->cur_tok = nullptr;
216 t->state = TokenizeStateStart;
217}
218
219static void end_symbol(Tokenize *t) {
220 put_back(t, 1);
221 end_token(t);
222 t->state = TokenizeStateStart;
223}
224
208static ZigList<Token> *tokenize(Buf *buf) {225static ZigList<Token> *tokenize(Buf *buf) {
209 Tokenize t = {0};226 Tokenize t = {0};
210 t.tokens = allocate<ZigList<Token>>(1);227 t.tokens = allocate<ZigList<Token>>(1);
211 for (t.pos = 0; t.pos < buf->len; t.pos += 1) {228 for (t.pos = 0; t.pos < buf_len(buf); t.pos += 1) {
212 uint8_t c = buf->ptr[t.pos];229 uint8_t c = buf_ptr(buf)[t.pos];
213 switch (t.state) {230 switch (t.state) {
214 case TokenizeStateStart:231 case TokenizeStateStart:
215 switch (c) {232 switch (c) {
...@@ -232,7 +249,7 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -232,7 +249,7 @@ static ZigList<Token> *tokenize(Buf *buf) {
232 end_token(&t);249 end_token(&t);
233 break;250 break;
234 case ')':251 case ')':
235 begin_token(&t, TokenIdLParen);252 begin_token(&t, TokenIdRParen);
236 end_token(&t);253 end_token(&t);
237 break;254 break;
238 case ',':255 case ',':
...@@ -269,22 +286,15 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -269,22 +286,15 @@ static ZigList<Token> *tokenize(Buf *buf) {
269 break;286 break;
270 case TokenizeStateDirective:287 case TokenizeStateDirective:
271 if (c == '\n') {288 if (c == '\n') {
272 assert(t.cur_tok);289 end_directive(&t);
273 t.cur_tok->end_pos = t.pos;
274 t.cur_tok = nullptr;
275 t.state = TokenizeStateStart;
276 }290 }
277 break;291 break;
278 case TokenizeStateSymbol:292 case TokenizeStateSymbol:
279 switch (c) {293 switch (c) {
280 case ALPHA:294 case SYMBOL_CHAR:
281 case DIGIT:
282 case '_':
283 break;295 break;
284 default:296 default:
285 put_back(&t, 1);297 end_symbol(&t);
286 end_token(&t);
287 t.state = TokenizeStateStart;
288 break;298 break;
289 }299 }
290 break;300 break;
...@@ -303,9 +313,7 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -303,9 +313,7 @@ static ZigList<Token> *tokenize(Buf *buf) {
303 case DIGIT:313 case DIGIT:
304 break;314 break;
305 default:315 default:
306 put_back(&t, 1);316 end_symbol(&t);
307 end_token(&t);
308 t.state = TokenizeStateStart;
309 break;317 break;
310 }318 }
311 break;319 break;
...@@ -317,6 +325,24 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -317,6 +325,24 @@ static ZigList<Token> *tokenize(Buf *buf) {
317 t.column += 1;325 t.column += 1;
318 }326 }
319 }327 }
328 // EOF
329 switch (t.state) {
330 case TokenizeStateStart:
331 break;
332 case TokenizeStateDirective:
333 end_directive(&t);
334 break;
335 case TokenizeStateSymbol:
336 end_symbol(&t);
337 break;
338 case TokenizeStateString:
339 tokenize_error(&t, "unterminated string");
340 break;
341 case TokenizeStateNumber:
342 end_symbol(&t);
343 break;
344 }
345 assert(!t.cur_tok);
320 return t.tokens;346 return t.tokens;
321}347}
322348
...@@ -342,11 +368,118 @@ static void print_tokens(Buf *buf, ZigList<Token> *tokens) {...@@ -342,11 +368,118 @@ static void print_tokens(Buf *buf, ZigList<Token> *tokens) {
342 for (int i = 0; i < tokens->length; i += 1) {368 for (int i = 0; i < tokens->length; i += 1) {
343 Token *token = &tokens->at(i);369 Token *token = &tokens->at(i);
344 printf("%s ", token_name(token));370 printf("%s ", token_name(token));
345 fwrite(buf->ptr + token->start_pos, 1, token->end_pos - token->start_pos, stdout);371 fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stdout);
346 printf("\n");372 printf("\n");
347 }373 }
348}374}
349375
376struct Preprocess {
377 Buf *out_buf;
378 Buf *in_buf;
379 Token *token;
380};
381
382__attribute__ ((format (printf, 2, 3)))
383static void preprocess_error(Preprocess *p, const char *format, ...) {
384 va_list ap;
385 va_start(ap, format);
386 fprintf(stderr, "Error: Line %d, column %d: ", p->token->start_line + 1, p->token->start_column + 1);
387 vfprintf(stderr, format, ap);
388 fprintf(stderr, "\n");
389 va_end(ap);
390 exit(EXIT_FAILURE);
391}
392
393enum IncludeState {
394 IncludeStateStart,
395 IncludeStateQuote,
396};
397
398static void render_include(Preprocess *p, Buf *include_path, char unquote_char) {
399 fprintf(stderr, "render_include \"%s\" '%c'\n", buf_ptr(include_path), unquote_char);
400}
401
402static void parse_and_render_include(Preprocess *p, Buf *directive_buf, int pos) {
403 int state = IncludeStateStart;
404 char unquote_char;
405 int quote_start_pos;
406 for (; pos < buf_len(directive_buf); pos += 1) {
407 uint8_t c = buf_ptr(directive_buf)[pos];
408 switch (state) {
409 case IncludeStateStart:
410 switch (c) {
411 case WHITESPACE:
412 break;
413 case '<':
414 case '"':
415 state = IncludeStateQuote;
416 quote_start_pos = pos;
417 unquote_char = (c == '<') ? '>' : '"';
418 break;
419
420 }
421 break;
422 case IncludeStateQuote:
423 if (c == unquote_char) {
424 Buf *include_path = buf_slice(directive_buf, quote_start_pos + 1, pos);
425 render_include(p, include_path, unquote_char);
426 return;
427 }
428 break;
429 }
430 }
431 preprocess_error(p, "include directive missing path");
432}
433
434static void render_directive(Preprocess *p, Buf *directive_buf) {
435 for (int pos = 1; pos < buf_len(directive_buf); pos += 1) {
436 uint8_t c = buf_ptr(directive_buf)[pos];
437 switch (c) {
438 case SYMBOL_CHAR:
439 break;
440 default:
441 pos -= 1;
442 Buf *directive_name = buf_from_mem(buf_ptr(directive_buf) + 1, pos);
443 if (strcmp(buf_ptr(directive_name), "include") == 0) {
444 parse_and_render_include(p, directive_buf, pos);
445 } else {
446 preprocess_error(p, "invalid directive: \"%s\"", buf_ptr(directive_name));
447 }
448 return;
449 }
450 }
451}
452
453static void render_token(Preprocess *p) {
454 Buf *token_buf = buf_slice(p->in_buf, p->token->start_pos, p->token->end_pos);
455 switch (p->token->id) {
456 case TokenIdDirective:
457 render_directive(p, token_buf);
458 break;
459 default:
460 buf_append_buf(p->out_buf, token_buf);
461 if (p->token->id == TokenIdSemicolon ||
462 p->token->id == TokenIdLBrace ||
463 p->token->id == TokenIdRBrace)
464 {
465 buf_append_str(p->out_buf, "\n", -1);
466 } else {
467 buf_append_str(p->out_buf, " ", -1);
468 }
469 }
470}
471
472static Buf *preprocess(Buf *in_buf, ZigList<Token> *tokens) {
473 Preprocess p = {0};
474 p.out_buf = buf_alloc();
475 p.in_buf = in_buf;
476 for (int i = 0; i < tokens->length; i += 1) {
477 p.token = &tokens->at(i);
478 render_token(&p);
479 }
480 return p.out_buf;
481}
482
350int main(int argc, char **argv) {483int main(int argc, char **argv) {
351 char *arg0 = argv[0];484 char *arg0 = argv[0];
352 char *in_file = NULL;485 char *in_file = NULL;
...@@ -386,14 +519,19 @@ int main(int argc, char **argv) {...@@ -386,14 +519,19 @@ int main(int argc, char **argv) {
386 zig_panic("unable to open %s for reading: %s\n", in_file, strerror(errno));519 zig_panic("unable to open %s for reading: %s\n", in_file, strerror(errno));
387 }520 }
388521
389 struct Buf *in_data = fetch_file(in_f);522 Buf *in_data = fetch_file(in_f);
390523
391 fprintf(stderr, "%s\n", in_data->ptr);524 fprintf(stderr, "Original source:\n%s\n", buf_ptr(in_data));
392525
393 ZigList<Token> *tokens = tokenize(in_data);526 ZigList<Token> *tokens = tokenize(in_data);
394527
528 fprintf(stderr, "\nTokens:\n");
395 print_tokens(in_data, tokens);529 print_tokens(in_data, tokens);
396530
531 Buf *preprocessed_source = preprocess(in_data, tokens);
532
533 fprintf(stderr, "\nPreprocessed source:\n%s\n", buf_ptr(preprocessed_source));
534
397535
398 return EXIT_SUCCESS;536 return EXIT_SUCCESS;
399}537}