authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-23 22:45:20-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-23 22:45:20-07:00
log5f48463bdd843a9d8dfd55cfc389637b39fc1074
tree387cc0442c09504dbdb6cd97c43c5a3d0e9da7fc
parentd519ce87dd3c112dbaa087e2ef08514d5253f02a

some half done thing I forgot about ¯\_(ツ)_/¯

2 files changed, 187 insertions(+), 160 deletions(-)

src/buffer.hpp+37-7
...@@ -73,18 +73,48 @@ static inline Buf *buf_slice(Buf *in_buf, int start, int end) {...@@ -73,18 +73,48 @@ static inline Buf *buf_slice(Buf *in_buf, int start, int end) {
73 return out_buf;73 return out_buf;
74}74}
7575
76static inline void buf_append_str(Buf *buf, const char *str, int str_len) {76static inline void buf_append_mem(Buf *buf, const char *mem, int mem_len) {
77 if (str_len == -1)77 assert(mem_len >= 0);
78 str_len = strlen(str);
79
80 int old_len = buf_len(buf);78 int old_len = buf_len(buf);
81 buf_resize(buf, old_len + str_len);79 buf_resize(buf, old_len + mem_len);
82 memcpy(buf_ptr(buf) + old_len, str, str_len);80 memcpy(buf_ptr(buf) + old_len, mem, mem_len);
83 buf->list.at(buf_len(buf)) = 0;81 buf->list.at(buf_len(buf)) = 0;
84}82}
8583
84static inline void buf_append_str(Buf *buf, const char *str) {
85 buf_append_mem(buf, str, strlen(str));
86}
87
86static inline void buf_append_buf(Buf *buf, Buf *append_buf) {88static inline void buf_append_buf(Buf *buf, Buf *append_buf) {
87 buf_append_str(buf, buf_ptr(append_buf), buf_len(append_buf));89 buf_append_mem(buf, buf_ptr(append_buf), buf_len(append_buf));
90}
91
92static inline void buf_append_char(Buf *buf, uint8_t c) {
93 buf_append_mem(buf, (const char *)&c, 1);
94}
95
96static inline bool buf_eql_mem(Buf *buf, const char *mem, int mem_len) {
97 if (buf_len(buf) != mem_len)
98 return false;
99 return memcmp(buf_ptr(buf), mem, mem_len) == 0;
100}
101
102static inline bool buf_eql_str(Buf *buf, const char *str) {
103 return buf_eql_mem(buf, str, strlen(str));
104}
105
106static inline bool buf_eql_buf(Buf *buf, Buf *other) {
107 return buf_eql_mem(buf, buf_ptr(other), buf_len(other));
108}
109
110static inline void buf_splice_buf(Buf *buf, int start, int end, Buf *other) {
111 if (start != end)
112 zig_panic("TODO buf_splice_buf");
113
114 int old_buf_len = buf_len(buf);
115 buf_resize(buf, old_buf_len + buf_len(other));
116 memmove(buf_ptr(buf) + start + buf_len(other), buf_ptr(buf) + start, old_buf_len - start);
117 memcpy(buf_ptr(buf) + start, buf_ptr(other), buf_len(other));
88}118}
89119
90// TODO this method needs work120// TODO this method needs work
src/main.cpp+150-153
...@@ -154,18 +154,29 @@ struct Token {...@@ -154,18 +154,29 @@ struct Token {
154enum TokenizeState {154enum TokenizeState {
155 TokenizeStateStart,155 TokenizeStateStart,
156 TokenizeStateDirective,156 TokenizeStateDirective,
157 TokenizeStateDirectiveName,
158 TokenizeStateIncludeQuote,
159 TokenizeStateDirectiveEnd,
160 TokenizeStateInclude,
157 TokenizeStateSymbol,161 TokenizeStateSymbol,
158 TokenizeStateString,162 TokenizeStateString,
159 TokenizeStateNumber,163 TokenizeStateNumber,
160};164};
161165
162struct Tokenize {166struct Tokenize {
167 Buf *buf;
163 int pos;168 int pos;
164 TokenizeState state;169 TokenizeState state;
165 ZigList<Token> *tokens;170 ZigList<Token> *tokens;
166 int line;171 int line;
167 int column;172 int column;
168 Token *cur_tok;173 Token *cur_tok;
174 Buf *directive_name;
175 Buf *cur_dir_path;
176 uint8_t unquote_char;
177 int quote_start_pos;
178 Buf *include_path;
179 ZigList<char *> *include_paths;
169};180};
170181
171__attribute__ ((format (printf, 2, 3)))182__attribute__ ((format (printf, 2, 3)))
...@@ -210,24 +221,72 @@ static void put_back(Tokenize *t, int count) {...@@ -210,24 +221,72 @@ static void put_back(Tokenize *t, int count) {
210 t->pos -= count;221 t->pos -= count;
211}222}
212223
224static void begin_directive(Tokenize *t) {
225 t->state = TokenizeStateDirective;
226 begin_token(t, TokenIdDirective);
227 assert(!t->directive_name);
228 t->directive_name = buf_alloc();
229}
230
231static bool find_and_include_file(Tokenize *t, char *dir_path, char *file_path) {
232 Buf *full_path = buf_sprintf("%s/%s", dir_path, file_path);
233
234 FILE *f = fopen(buf_ptr(full_path), "rb");
235 if (!f)
236 return false;
237
238 Buf *contents = fetch_file(f);
239
240 buf_splice_buf(t->buf, t->pos, t->pos, contents);
241
242 return true;
243}
244
245static void render_include(Tokenize *t, Buf *target_path, char unquote_char) {
246 if (unquote_char == '"') {
247 if (find_and_include_file(t, buf_ptr(t->cur_dir_path), buf_ptr(target_path)))
248 return;
249 }
250 for (int i = 0; i < t->include_paths->length; i += 1) {
251 char *include_path = t->include_paths->at(i);
252 if (find_and_include_file(t, include_path, buf_ptr(target_path)))
253 return;
254 }
255 tokenize_error(t, "include path \"%s\" not found", buf_ptr(target_path));
256}
257
213static void end_directive(Tokenize *t) {258static void end_directive(Tokenize *t) {
214 assert(t->cur_tok);259 end_token(t);
215 t->cur_tok->end_pos = t->pos;260 if (t->include_path) {
216 t->cur_tok = nullptr;261 render_include(t, t->include_path, t->unquote_char);
262 t->include_path = nullptr;
263 }
217 t->state = TokenizeStateStart;264 t->state = TokenizeStateStart;
218}265}
219266
267static void end_directive_name(Tokenize *t) {
268 if (buf_eql_str(t->directive_name, "include")) {
269 t->state = TokenizeStateInclude;
270 t->directive_name = nullptr;
271 } else {
272 tokenize_error(t, "invalid directive name: \"%s\"", buf_ptr(t->directive_name));
273 }
274}
275
220static void end_symbol(Tokenize *t) {276static void end_symbol(Tokenize *t) {
221 put_back(t, 1);277 put_back(t, 1);
222 end_token(t);278 end_token(t);
223 t->state = TokenizeStateStart;279 t->state = TokenizeStateStart;
224}280}
225281
226static ZigList<Token> *tokenize(Buf *buf) {282static ZigList<Token> *tokenize(Buf *buf, ZigList<char *> *include_paths, Buf *cur_dir_path) {
227 Tokenize t = {0};283 Tokenize t = {0};
228 t.tokens = allocate<ZigList<Token>>(1);284 t.tokens = allocate<ZigList<Token>>(1);
229 for (t.pos = 0; t.pos < buf_len(buf); t.pos += 1) {285 t.buf = buf;
230 uint8_t c = buf_ptr(buf)[t.pos];286 t.cur_dir_path = cur_dir_path;
287 t.include_paths = include_paths;
288 for (t.pos = 0; t.pos < buf_len(t.buf); t.pos += 1) {
289 uint8_t c = buf_ptr(t.buf)[t.pos];
231 switch (t.state) {290 switch (t.state) {
232 case TokenizeStateStart:291 case TokenizeStateStart:
233 switch (c) {292 switch (c) {
...@@ -242,8 +301,11 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -242,8 +301,11 @@ static ZigList<Token> *tokenize(Buf *buf) {
242 begin_token(&t, TokenIdNumberLiteral);301 begin_token(&t, TokenIdNumberLiteral);
243 break;302 break;
244 case '#':303 case '#':
245 t.state = TokenizeStateDirective;304 begin_directive(&t);
246 begin_token(&t, TokenIdDirective);305 break;
306 case '"':
307 begin_token(&t, TokenIdStringLiteral);
308 t.state = TokenizeStateString;
247 break;309 break;
248 case '(':310 case '(':
249 begin_token(&t, TokenIdLParen);311 begin_token(&t, TokenIdLParen);
...@@ -269,10 +331,6 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -269,10 +331,6 @@ static ZigList<Token> *tokenize(Buf *buf) {
269 begin_token(&t, TokenIdRBrace);331 begin_token(&t, TokenIdRBrace);
270 end_token(&t);332 end_token(&t);
271 break;333 break;
272 case '"':
273 begin_token(&t, TokenIdStringLiteral);
274 t.state = TokenizeStateString;
275 break;
276 case ';':334 case ';':
277 begin_token(&t, TokenIdSemicolon);335 begin_token(&t, TokenIdSemicolon);
278 end_token(&t);336 end_token(&t);
...@@ -286,8 +344,70 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -286,8 +344,70 @@ static ZigList<Token> *tokenize(Buf *buf) {
286 }344 }
287 break;345 break;
288 case TokenizeStateDirective:346 case TokenizeStateDirective:
289 if (c == '\n') {347 switch (c) {
290 end_directive(&t);348 case '\n':
349 end_directive_name(&t);
350 end_directive(&t);
351 break;
352 case ' ':
353 case '\t':
354 case '\f':
355 case '\r':
356 case 0xb:
357 break;
358 case SYMBOL_CHAR:
359 t.state = TokenizeStateDirectiveName;
360 buf_append_char(t.directive_name, c);
361 break;
362 default:
363 tokenize_error(&t, "invalid directive character: '%c'", c);
364 break;
365 }
366 break;
367 case TokenizeStateDirectiveName:
368 switch (c) {
369 case WHITESPACE:
370 end_directive_name(&t);
371 break;
372 case SYMBOL_CHAR:
373 buf_append_char(t.directive_name, c);
374 break;
375 default:
376 tokenize_error(&t, "invalid directive name character: '%c'", c);
377 break;
378 }
379 break;
380 case TokenizeStateInclude:
381 switch (c) {
382 case WHITESPACE:
383 break;
384 case '<':
385 case '"':
386 t.state = TokenizeStateIncludeQuote;
387 t.quote_start_pos = t.pos;
388 t.unquote_char = (c == '<') ? '>' : '"';
389 break;
390 }
391 break;
392 case TokenizeStateIncludeQuote:
393 if (c == t.unquote_char) {
394 t.include_path = buf_slice(t.buf, t.quote_start_pos + 1, t.pos);
395 t.state = TokenizeStateDirectiveEnd;
396 }
397 break;
398 case TokenizeStateDirectiveEnd:
399 switch (c) {
400 case '\n':
401 end_directive(&t);
402 break;
403 case ' ':
404 case '\t':
405 case '\f':
406 case '\r':
407 case 0xb:
408 break;
409 default:
410 tokenize_error(&t, "expected whitespace or newline: '%c'", c);
291 }411 }
292 break;412 break;
293 case TokenizeStateSymbol:413 case TokenizeStateSymbol:
...@@ -333,6 +453,13 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -333,6 +453,13 @@ static ZigList<Token> *tokenize(Buf *buf) {
333 case TokenizeStateDirective:453 case TokenizeStateDirective:
334 end_directive(&t);454 end_directive(&t);
335 break;455 break;
456 case TokenizeStateDirectiveName:
457 end_directive_name(&t);
458 end_directive(&t);
459 break;
460 case TokenizeStateInclude:
461 tokenize_error(&t, "missing include path");
462 break;
336 case TokenizeStateSymbol:463 case TokenizeStateSymbol:
337 end_symbol(&t);464 end_symbol(&t);
338 break;465 break;
...@@ -342,6 +469,12 @@ static ZigList<Token> *tokenize(Buf *buf) {...@@ -342,6 +469,12 @@ static ZigList<Token> *tokenize(Buf *buf) {
342 case TokenizeStateNumber:469 case TokenizeStateNumber:
343 end_symbol(&t);470 end_symbol(&t);
344 break;471 break;
472 case TokenizeStateIncludeQuote:
473 tokenize_error(&t, "unterminated include path");
474 break;
475 case TokenizeStateDirectiveEnd:
476 end_directive(&t);
477 break;
345 }478 }
346 assert(!t.cur_tok);479 assert(!t.cur_tok);
347 return t.tokens;480 return t.tokens;
...@@ -374,144 +507,6 @@ static void print_tokens(Buf *buf, ZigList<Token> *tokens) {...@@ -374,144 +507,6 @@ static void print_tokens(Buf *buf, ZigList<Token> *tokens) {
374 }507 }
375}508}
376509
377struct Preprocess {
378 Buf *out_buf;
379 Buf *in_buf;
380 Token *token;
381 ZigList<char *> *include_paths;
382 Buf *cur_dir_path;
383};
384
385__attribute__ ((format (printf, 2, 3)))
386static void preprocess_error(Preprocess *p, const char *format, ...) {
387 va_list ap;
388 va_start(ap, format);
389 fprintf(stderr, "Error: Line %d, column %d: ", p->token->start_line + 1, p->token->start_column + 1);
390 vfprintf(stderr, format, ap);
391 fprintf(stderr, "\n");
392 va_end(ap);
393 exit(EXIT_FAILURE);
394}
395
396enum IncludeState {
397 IncludeStateStart,
398 IncludeStateQuote,
399};
400
401static Buf *find_include_file(Preprocess *p, char *dir_path, char *file_path) {
402 Buf *full_path = buf_sprintf("%s/%s", dir_path, file_path);
403
404 FILE *f = fopen(buf_ptr(full_path), "rb");
405 if (!f)
406 return nullptr;
407
408 return fetch_file(f);
409}
410
411static void render_include(Preprocess *p, Buf *target_path, char unquote_char) {
412 if (unquote_char == '"') {
413 Buf *file_contents = find_include_file(p, buf_ptr(p->cur_dir_path), buf_ptr(target_path));
414 if (file_contents) {
415 buf_append_buf(p->out_buf, file_contents);
416 return;
417 }
418 }
419 for (int i = 0; i < p->include_paths->length; i += 1) {
420 char *include_path = p->include_paths->at(i);
421 Buf *file_contents = find_include_file(p, include_path, buf_ptr(target_path));
422 if (file_contents) {
423 buf_append_buf(p->out_buf, file_contents);
424 return;
425 }
426 }
427 preprocess_error(p, "include path \"%s\" not found", buf_ptr(target_path));
428}
429
430static void parse_and_render_include(Preprocess *p, Buf *directive_buf, int pos) {
431 int state = IncludeStateStart;
432 char unquote_char;
433 int quote_start_pos;
434 for (; pos < buf_len(directive_buf); pos += 1) {
435 uint8_t c = buf_ptr(directive_buf)[pos];
436 switch (state) {
437 case IncludeStateStart:
438 switch (c) {
439 case WHITESPACE:
440 break;
441 case '<':
442 case '"':
443 state = IncludeStateQuote;
444 quote_start_pos = pos;
445 unquote_char = (c == '<') ? '>' : '"';
446 break;
447
448 }
449 break;
450 case IncludeStateQuote:
451 if (c == unquote_char) {
452 Buf *include_path = buf_slice(directive_buf, quote_start_pos + 1, pos);
453 render_include(p, include_path, unquote_char);
454 return;
455 }
456 break;
457 }
458 }
459 preprocess_error(p, "include directive missing path");
460}
461
462static void render_directive(Preprocess *p, Buf *directive_buf) {
463 for (int pos = 1; pos < buf_len(directive_buf); pos += 1) {
464 uint8_t c = buf_ptr(directive_buf)[pos];
465 switch (c) {
466 case SYMBOL_CHAR:
467 break;
468 default:
469 pos -= 1;
470 Buf *directive_name = buf_from_mem(buf_ptr(directive_buf) + 1, pos);
471 if (strcmp(buf_ptr(directive_name), "include") == 0) {
472 parse_and_render_include(p, directive_buf, pos);
473 } else {
474 preprocess_error(p, "invalid directive: \"%s\"", buf_ptr(directive_name));
475 }
476 return;
477 }
478 }
479}
480
481static void render_token(Preprocess *p) {
482 Buf *token_buf = buf_slice(p->in_buf, p->token->start_pos, p->token->end_pos);
483 switch (p->token->id) {
484 case TokenIdDirective:
485 render_directive(p, token_buf);
486 break;
487 default:
488 buf_append_buf(p->out_buf, token_buf);
489 if (p->token->id == TokenIdSemicolon ||
490 p->token->id == TokenIdLBrace ||
491 p->token->id == TokenIdRBrace)
492 {
493 buf_append_str(p->out_buf, "\n", -1);
494 } else {
495 buf_append_str(p->out_buf, " ", -1);
496 }
497 }
498}
499
500static Buf *preprocess(Buf *in_buf, ZigList<Token> *tokens,
501 ZigList<char *> *include_paths, Buf *cur_dir_path)
502{
503 Preprocess p = {0};
504 p.out_buf = buf_alloc();
505 p.in_buf = in_buf;
506 p.include_paths = include_paths;
507 p.cur_dir_path = cur_dir_path;
508 for (int i = 0; i < tokens->length; i += 1) {
509 p.token = &tokens->at(i);
510 render_token(&p);
511 }
512 return p.out_buf;
513}
514
515char cur_dir[1024];510char cur_dir[1024];
516511
517int main(int argc, char **argv) {512int main(int argc, char **argv) {
...@@ -566,14 +561,16 @@ int main(int argc, char **argv) {...@@ -566,14 +561,16 @@ int main(int argc, char **argv) {
566561
567 fprintf(stderr, "Original source:\n%s\n", buf_ptr(in_data));562 fprintf(stderr, "Original source:\n%s\n", buf_ptr(in_data));
568563
569 ZigList<Token> *tokens = tokenize(in_data);564 ZigList<Token> *tokens = tokenize(in_data, &include_paths, cur_dir_path);
570565
571 fprintf(stderr, "\nTokens:\n");566 fprintf(stderr, "\nTokens:\n");
572 print_tokens(in_data, tokens);567 print_tokens(in_data, tokens);
573568
569 /*
574 Buf *preprocessed_source = preprocess(in_data, tokens, &include_paths, cur_dir_path);570 Buf *preprocessed_source = preprocess(in_data, tokens, &include_paths, cur_dir_path);
575571
576 fprintf(stderr, "\nPreprocessed source:\n%s\n", buf_ptr(preprocessed_source));572 fprintf(stderr, "\nPreprocessed source:\n%s\n", buf_ptr(preprocessed_source));
573 */
577574
578575
579 return EXIT_SUCCESS;576 return EXIT_SUCCESS;