authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-05 20:26:58-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-05 20:26:58-07:00
loge71521335ac656f94892ee049cc4814ecb772b30
treebe4c24abec3ac3e434f70b6ba754e7d681dd8bd1
parente09932928ac16681d71127f3d5cc0c488f2fdcc5

preprocessor detects #include directives

3 files changed, 264 insertions(+), 44 deletions(-)

src/buffer.hpp created+82
......@@ -0,0 +1,82 @@
1/*
2 * Copyright (c) 2015 Andrew Kelley
3 *
4 * This file is part of zig, which is MIT licensed.
5 * See http://opensource.org/licenses/MIT
6 */
7
8#ifndef ZIG_BUFFER_HPP
9#define ZIG_BUFFER_HPP
10
11#include "list.hpp"
12
13#include <assert.h>
14
15struct Buf {
16 ZigList<char> list;
17};
18
19static inline int buf_len(Buf *buf) {
20 return buf->list.length - 1;
21}
22
23static inline char *buf_ptr(Buf *buf) {
24 return buf->list.items;
25}
26
27static inline void buf_resize(Buf *buf, int new_len) {
28 buf->list.resize(new_len + 1);
29 buf->list.at(buf_len(buf)) = 0;
30}
31
32static inline Buf *buf_alloc(void) {
33 Buf *buf = allocate<Buf>(1);
34 buf_resize(buf, 0);
35 return buf;
36}
37
38static inline Buf *buf_alloc_fixed(int size) {
39 Buf *buf = allocate<Buf>(1);
40 buf_resize(buf, size);
41 return buf;
42}
43
44static inline void buf_deinit(Buf *buf) {
45 buf->list.deinit();
46}
47
48static inline Buf *buf_from_mem(char *ptr, int len) {
49 Buf *buf = allocate<Buf>(1);
50 buf->list.resize(len + 1);
51 memcpy(buf_ptr(buf), ptr, len);
52 buf->list.at(buf_len(buf)) = 0;
53 return buf;
54}
55
56static inline Buf *buf_slice(Buf *in_buf, int start, int end) {
57 assert(start >= 0);
58 assert(end >= 0);
59 assert(start < buf_len(in_buf));
60 assert(end <= buf_len(in_buf));
61 Buf *out_buf = allocate<Buf>(1);
62 out_buf->list.resize(end - start + 1);
63 memcpy(buf_ptr(out_buf), buf_ptr(in_buf) + start, end - start);
64 out_buf->list.at(buf_len(out_buf)) = 0;
65 return out_buf;
66}
67
68static inline void buf_append_str(Buf *buf, const char *str, int str_len) {
69 if (str_len == -1)
70 str_len = strlen(str);
71
72 int old_len = buf_len(buf);
73 buf_resize(buf, old_len + str_len);
74 memcpy(buf_ptr(buf) + old_len, str, str_len);
75 buf->list.at(buf_len(buf)) = 0;
76}
77
78static inline void buf_append_buf(Buf *buf, Buf *append_buf) {
79 buf_append_str(buf, buf_ptr(append_buf), buf_len(append_buf));
80}
81
82#endif
src/list.hpp+1-1
......@@ -15,7 +15,7 @@
1515template<typename T>
1616struct ZigList {
1717 void deinit() {
18 deallocate(items);
18 free(items);
1919 }
2020 void append(T item) {
2121 ensure_capacity(length + 1);
src/main.cpp+181-43
......@@ -8,6 +8,8 @@
88#include "config.h"
99#include "util.hpp"
1010#include "list.hpp"
11#include "buffer.hpp"
12
1113#include <stdio.h>
1214#include <string.h>
1315#include <stdlib.h>
......@@ -19,25 +21,6 @@
1921#include <sys/stat.h>
2022#include <unistd.h>
2123
22struct Buf {
23 int len;
24 char ptr[0];
25};
26
27static Buf *alloc_buf(int size) {
28 Buf *buf = (Buf *)allocate_nonzero<char>(sizeof(Buf) + size + 1);
29 buf->len = size;
30 buf->ptr[buf->len] = 0;
31 return buf;
32}
33
34/*
35static void fprint_buf(FILE *f, Buf *buf) {
36 if (fwrite(buf->ptr, 1, buf->len, f))
37 zig_panic("error writing: %s", strerror(errno));
38}
39*/
40
4124static int usage(char *arg0) {
4225 fprintf(stderr, "Usage: %s --output outfile code.zig\n"
4326 "Other options:\n"
......@@ -46,7 +29,7 @@ static int usage(char *arg0) {
4629 return EXIT_FAILURE;
4730}
4831
49static struct Buf *fetch_file(FILE *f) {
32static Buf *fetch_file(FILE *f) {
5033 int fd = fileno(f);
5134 struct stat st;
5235 if (fstat(fd, &st))
......@@ -56,9 +39,9 @@ static struct Buf *fetch_file(FILE *f) {
5639 zig_panic("file too big");
5740 int size = (int)big_size;
5841
59 Buf *buf = alloc_buf(size);
60 size_t amt_read = fread(buf->ptr, 1, buf->len, f);
61 if (amt_read != (size_t)buf->len)
42 Buf *buf = buf_alloc_fixed(size);
43 size_t amt_read = fread(buf_ptr(buf), 1, buf_len(buf), f);
44 if (amt_read != (size_t)buf_len(buf))
6245 zig_panic("error reading: %s", strerror(errno));
6346
6447 return buf;
......@@ -138,6 +121,12 @@ static struct Buf *fetch_file(FILE *f) {
138121 case 'Y': \
139122 case 'Z'
140123
124#define SYMBOL_CHAR \
125 ALPHA: \
126 case DIGIT: \
127 case '_'
128
129
141130enum TokenId {
142131 TokenIdDirective,
143132 TokenIdSymbol,
......@@ -157,6 +146,8 @@ struct Token {
157146 TokenId id;
158147 int start_pos;
159148 int end_pos;
149 int start_line;
150 int start_column;
160151};
161152
162153enum TokenizeState {
......@@ -178,10 +169,21 @@ struct Tokenize {
178169
179170__attribute__ ((format (printf, 2, 3)))
180171static void tokenize_error(Tokenize *t, const char *format, ...) {
172 int line;
173 int column;
174 if (t->cur_tok) {
175 line = t->cur_tok->start_line + 1;
176 column = t->cur_tok->start_column + 1;
177 } else {
178 line = t->line + 1;
179 column = t->column + 1;
180 }
181
181182 va_list ap;
182183 va_start(ap, format);
183 fprintf(stderr, "Error. Line %d, column %d: ", t->line + 1, t->column + 1);
184 fprintf(stderr, "Error: Line %d, column %d: ", line, column);
184185 vfprintf(stderr, format, ap);
186 fprintf(stderr, "\n");
185187 va_end(ap);
186188 exit(EXIT_FAILURE);
187189}
......@@ -190,6 +192,8 @@ static void begin_token(Tokenize *t, TokenId id) {
190192 assert(!t->cur_tok);
191193 t->tokens->add_one();
192194 Token *token = &t->tokens->last();
195 token->start_line = t->line;
196 token->start_column = t->column;
193197 token->id = id;
194198 token->start_pos = t->pos;
195199 t->cur_tok = token;
......@@ -205,11 +209,24 @@ static void put_back(Tokenize *t, int count) {
205209 t->pos -= count;
206210}
207211
212static void end_directive(Tokenize *t) {
213 assert(t->cur_tok);
214 t->cur_tok->end_pos = t->pos;
215 t->cur_tok = nullptr;
216 t->state = TokenizeStateStart;
217}
218
219static void end_symbol(Tokenize *t) {
220 put_back(t, 1);
221 end_token(t);
222 t->state = TokenizeStateStart;
223}
224
208225static ZigList<Token> *tokenize(Buf *buf) {
209226 Tokenize t = {0};
210227 t.tokens = allocate<ZigList<Token>>(1);
211 for (t.pos = 0; t.pos < buf->len; t.pos += 1) {
212 uint8_t c = buf->ptr[t.pos];
228 for (t.pos = 0; t.pos < buf_len(buf); t.pos += 1) {
229 uint8_t c = buf_ptr(buf)[t.pos];
213230 switch (t.state) {
214231 case TokenizeStateStart:
215232 switch (c) {
......@@ -232,7 +249,7 @@ static ZigList<Token> *tokenize(Buf *buf) {
232249 end_token(&t);
233250 break;
234251 case ')':
235 begin_token(&t, TokenIdLParen);
252 begin_token(&t, TokenIdRParen);
236253 end_token(&t);
237254 break;
238255 case ',':
......@@ -269,22 +286,15 @@ static ZigList<Token> *tokenize(Buf *buf) {
269286 break;
270287 case TokenizeStateDirective:
271288 if (c == '\n') {
272 assert(t.cur_tok);
273 t.cur_tok->end_pos = t.pos;
274 t.cur_tok = nullptr;
275 t.state = TokenizeStateStart;
289 end_directive(&t);
276290 }
277291 break;
278292 case TokenizeStateSymbol:
279293 switch (c) {
280 case ALPHA:
281 case DIGIT:
282 case '_':
294 case SYMBOL_CHAR:
283295 break;
284296 default:
285 put_back(&t, 1);
286 end_token(&t);
287 t.state = TokenizeStateStart;
297 end_symbol(&t);
288298 break;
289299 }
290300 break;
......@@ -303,9 +313,7 @@ static ZigList<Token> *tokenize(Buf *buf) {
303313 case DIGIT:
304314 break;
305315 default:
306 put_back(&t, 1);
307 end_token(&t);
308 t.state = TokenizeStateStart;
316 end_symbol(&t);
309317 break;
310318 }
311319 break;
......@@ -317,6 +325,24 @@ static ZigList<Token> *tokenize(Buf *buf) {
317325 t.column += 1;
318326 }
319327 }
328 // EOF
329 switch (t.state) {
330 case TokenizeStateStart:
331 break;
332 case TokenizeStateDirective:
333 end_directive(&t);
334 break;
335 case TokenizeStateSymbol:
336 end_symbol(&t);
337 break;
338 case TokenizeStateString:
339 tokenize_error(&t, "unterminated string");
340 break;
341 case TokenizeStateNumber:
342 end_symbol(&t);
343 break;
344 }
345 assert(!t.cur_tok);
320346 return t.tokens;
321347}
322348
......@@ -342,11 +368,118 @@ static void print_tokens(Buf *buf, ZigList<Token> *tokens) {
342368 for (int i = 0; i < tokens->length; i += 1) {
343369 Token *token = &tokens->at(i);
344370 printf("%s ", token_name(token));
345 fwrite(buf->ptr + token->start_pos, 1, token->end_pos - token->start_pos, stdout);
371 fwrite(buf_ptr(buf) + token->start_pos, 1, token->end_pos - token->start_pos, stdout);
346372 printf("\n");
347373 }
348374}
349375
376struct Preprocess {
377 Buf *out_buf;
378 Buf *in_buf;
379 Token *token;
380};
381
382__attribute__ ((format (printf, 2, 3)))
383static void preprocess_error(Preprocess *p, const char *format, ...) {
384 va_list ap;
385 va_start(ap, format);
386 fprintf(stderr, "Error: Line %d, column %d: ", p->token->start_line + 1, p->token->start_column + 1);
387 vfprintf(stderr, format, ap);
388 fprintf(stderr, "\n");
389 va_end(ap);
390 exit(EXIT_FAILURE);
391}
392
393enum IncludeState {
394 IncludeStateStart,
395 IncludeStateQuote,
396};
397
398static void render_include(Preprocess *p, Buf *include_path, char unquote_char) {
399 fprintf(stderr, "render_include \"%s\" '%c'\n", buf_ptr(include_path), unquote_char);
400}
401
402static void parse_and_render_include(Preprocess *p, Buf *directive_buf, int pos) {
403 int state = IncludeStateStart;
404 char unquote_char;
405 int quote_start_pos;
406 for (; pos < buf_len(directive_buf); pos += 1) {
407 uint8_t c = buf_ptr(directive_buf)[pos];
408 switch (state) {
409 case IncludeStateStart:
410 switch (c) {
411 case WHITESPACE:
412 break;
413 case '<':
414 case '"':
415 state = IncludeStateQuote;
416 quote_start_pos = pos;
417 unquote_char = (c == '<') ? '>' : '"';
418 break;
419
420 }
421 break;
422 case IncludeStateQuote:
423 if (c == unquote_char) {
424 Buf *include_path = buf_slice(directive_buf, quote_start_pos + 1, pos);
425 render_include(p, include_path, unquote_char);
426 return;
427 }
428 break;
429 }
430 }
431 preprocess_error(p, "include directive missing path");
432}
433
434static void render_directive(Preprocess *p, Buf *directive_buf) {
435 for (int pos = 1; pos < buf_len(directive_buf); pos += 1) {
436 uint8_t c = buf_ptr(directive_buf)[pos];
437 switch (c) {
438 case SYMBOL_CHAR:
439 break;
440 default:
441 pos -= 1;
442 Buf *directive_name = buf_from_mem(buf_ptr(directive_buf) + 1, pos);
443 if (strcmp(buf_ptr(directive_name), "include") == 0) {
444 parse_and_render_include(p, directive_buf, pos);
445 } else {
446 preprocess_error(p, "invalid directive: \"%s\"", buf_ptr(directive_name));
447 }
448 return;
449 }
450 }
451}
452
453static void render_token(Preprocess *p) {
454 Buf *token_buf = buf_slice(p->in_buf, p->token->start_pos, p->token->end_pos);
455 switch (p->token->id) {
456 case TokenIdDirective:
457 render_directive(p, token_buf);
458 break;
459 default:
460 buf_append_buf(p->out_buf, token_buf);
461 if (p->token->id == TokenIdSemicolon ||
462 p->token->id == TokenIdLBrace ||
463 p->token->id == TokenIdRBrace)
464 {
465 buf_append_str(p->out_buf, "\n", -1);
466 } else {
467 buf_append_str(p->out_buf, " ", -1);
468 }
469 }
470}
471
472static Buf *preprocess(Buf *in_buf, ZigList<Token> *tokens) {
473 Preprocess p = {0};
474 p.out_buf = buf_alloc();
475 p.in_buf = in_buf;
476 for (int i = 0; i < tokens->length; i += 1) {
477 p.token = &tokens->at(i);
478 render_token(&p);
479 }
480 return p.out_buf;
481}
482
350483int main(int argc, char **argv) {
351484 char *arg0 = argv[0];
352485 char *in_file = NULL;
......@@ -386,14 +519,19 @@ int main(int argc, char **argv) {
386519 zig_panic("unable to open %s for reading: %s\n", in_file, strerror(errno));
387520 }
388521
389 struct Buf *in_data = fetch_file(in_f);
522 Buf *in_data = fetch_file(in_f);
390523
391 fprintf(stderr, "%s\n", in_data->ptr);
524 fprintf(stderr, "Original source:\n%s\n", buf_ptr(in_data));
392525
393526 ZigList<Token> *tokens = tokenize(in_data);
394527
528 fprintf(stderr, "\nTokens:\n");
395529 print_tokens(in_data, tokens);
396530
531 Buf *preprocessed_source = preprocess(in_data, tokens);
532
533 fprintf(stderr, "\nPreprocessed source:\n%s\n", buf_ptr(preprocessed_source));
534
397535
398536 return EXIT_SUCCESS;
399537}