authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-05 17:44:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-05 17:44:05-07:00
loge09932928ac16681d71127f3d5cc0c488f2fdcc5
tree57ded4c21186e13a5e0283a6e5fa30bbe07738f6
parent899c9fe94e04f50c6176644b61e2d89cb74a8886

tokenize

4 files changed, 318 insertions(+), 13 deletions(-)

README.md+4
......@@ -1 +1,5 @@
11# zig lang
2
3C upgrade.
4
5Start with C.
src/list.hpp+5-12
......@@ -5,24 +5,21 @@
55 * See http://opensource.org/licenses/MIT
66 */
77
8#ifndef GROOVE_LIST_HPP
9#define GROOVE_LIST_HPP
8#ifndef ZIG_LIST_HPP
9#define ZIG_LIST_HPP
1010
1111#include "util.hpp"
1212
1313#include <assert.h>
1414
1515template<typename T>
16struct GrooveList {
16struct ZigList {
1717 void deinit() {
1818 deallocate(items);
1919 }
2020 void append(T item) {
21 int err = ensure_capacity(length + 1);
22 if (err)
23 return err;
21 ensure_capacity(length + 1);
2422 items[length++] = item;
25 return 0;
2623 }
2724 // remember that the pointer to this item is invalid after you
2825 // modify the length of the list
......@@ -57,11 +54,8 @@ struct GrooveList {
5754
5855 void resize(int new_length) {
5956 assert(new_length >= 0);
60 int err = ensure_capacity(new_length);
61 if (err)
62 return err;
57 ensure_capacity(new_length);
6358 length = new_length;
64 return 0;
6559 }
6660
6761 void clear() {
......@@ -76,7 +70,6 @@ struct GrooveList {
7670 items = reallocate_nonzero(items, better_capacity);
7771 capacity = better_capacity;
7872 }
79 return 0;
8073 }
8174
8275 T * items;
src/main.cpp+294-1
......@@ -7,6 +7,7 @@
77
88#include "config.h"
99#include "util.hpp"
10#include "list.hpp"
1011#include <stdio.h>
1112#include <string.h>
1213#include <stdlib.h>
......@@ -30,6 +31,13 @@ static Buf *alloc_buf(int size) {
3031 return buf;
3132}
3233
34/*
35static void fprint_buf(FILE *f, Buf *buf) {
36 if (fwrite(buf->ptr, 1, buf->len, f))
37 zig_panic("error writing: %s", strerror(errno));
38}
39*/
40
3341static int usage(char *arg0) {
3442 fprintf(stderr, "Usage: %s --output outfile code.zig\n"
3543 "Other options:\n"
......@@ -56,6 +64,289 @@ static struct Buf *fetch_file(FILE *f) {
5664 return buf;
5765}
5866
67#define WHITESPACE \
68 ' ': \
69 case '\t': \
70 case '\n': \
71 case '\f': \
72 case '\r': \
73 case 0xb
74
75#define DIGIT \
76 '0': \
77 case '1': \
78 case '2': \
79 case '3': \
80 case '4': \
81 case '5': \
82 case '6': \
83 case '7': \
84 case '8': \
85 case '9'
86
87#define ALPHA \
88 'a': \
89 case 'b': \
90 case 'c': \
91 case 'd': \
92 case 'e': \
93 case 'f': \
94 case 'g': \
95 case 'h': \
96 case 'i': \
97 case 'j': \
98 case 'k': \
99 case 'l': \
100 case 'm': \
101 case 'n': \
102 case 'o': \
103 case 'p': \
104 case 'q': \
105 case 'r': \
106 case 's': \
107 case 't': \
108 case 'u': \
109 case 'v': \
110 case 'w': \
111 case 'x': \
112 case 'y': \
113 case 'z': \
114 case 'A': \
115 case 'B': \
116 case 'C': \
117 case 'D': \
118 case 'E': \
119 case 'F': \
120 case 'G': \
121 case 'H': \
122 case 'I': \
123 case 'J': \
124 case 'K': \
125 case 'L': \
126 case 'M': \
127 case 'N': \
128 case 'O': \
129 case 'P': \
130 case 'Q': \
131 case 'R': \
132 case 'S': \
133 case 'T': \
134 case 'U': \
135 case 'V': \
136 case 'W': \
137 case 'X': \
138 case 'Y': \
139 case 'Z'
140
141enum TokenId {
142 TokenIdDirective,
143 TokenIdSymbol,
144 TokenIdLParen,
145 TokenIdRParen,
146 TokenIdComma,
147 TokenIdStar,
148 TokenIdLBrace,
149 TokenIdRBrace,
150 TokenIdStringLiteral,
151 TokenIdSemicolon,
152 TokenIdNumberLiteral,
153 TokenIdPlus,
154};
155
156struct Token {
157 TokenId id;
158 int start_pos;
159 int end_pos;
160};
161
162enum TokenizeState {
163 TokenizeStateStart,
164 TokenizeStateDirective,
165 TokenizeStateSymbol,
166 TokenizeStateString,
167 TokenizeStateNumber,
168};
169
170struct Tokenize {
171 int pos;
172 TokenizeState state;
173 ZigList<Token> *tokens;
174 int line;
175 int column;
176 Token *cur_tok;
177};
178
179__attribute__ ((format (printf, 2, 3)))
180static void tokenize_error(Tokenize *t, const char *format, ...) {
181 va_list ap;
182 va_start(ap, format);
183 fprintf(stderr, "Error. Line %d, column %d: ", t->line + 1, t->column + 1);
184 vfprintf(stderr, format, ap);
185 va_end(ap);
186 exit(EXIT_FAILURE);
187}
188
189static void begin_token(Tokenize *t, TokenId id) {
190 assert(!t->cur_tok);
191 t->tokens->add_one();
192 Token *token = &t->tokens->last();
193 token->id = id;
194 token->start_pos = t->pos;
195 t->cur_tok = token;
196}
197
198static void end_token(Tokenize *t) {
199 assert(t->cur_tok);
200 t->cur_tok->end_pos = t->pos + 1;
201 t->cur_tok = nullptr;
202}
203
204static void put_back(Tokenize *t, int count) {
205 t->pos -= count;
206}
207
208static ZigList<Token> *tokenize(Buf *buf) {
209 Tokenize t = {0};
210 t.tokens = allocate<ZigList<Token>>(1);
211 for (t.pos = 0; t.pos < buf->len; t.pos += 1) {
212 uint8_t c = buf->ptr[t.pos];
213 switch (t.state) {
214 case TokenizeStateStart:
215 switch (c) {
216 case WHITESPACE:
217 break;
218 case ALPHA:
219 t.state = TokenizeStateSymbol;
220 begin_token(&t, TokenIdSymbol);
221 break;
222 case DIGIT:
223 t.state = TokenizeStateNumber;
224 begin_token(&t, TokenIdNumberLiteral);
225 break;
226 case '#':
227 t.state = TokenizeStateDirective;
228 begin_token(&t, TokenIdDirective);
229 break;
230 case '(':
231 begin_token(&t, TokenIdLParen);
232 end_token(&t);
233 break;
234 case ')':
235 begin_token(&t, TokenIdLParen);
236 end_token(&t);
237 break;
238 case ',':
239 begin_token(&t, TokenIdComma);
240 end_token(&t);
241 break;
242 case '*':
243 begin_token(&t, TokenIdStar);
244 end_token(&t);
245 break;
246 case '{':
247 begin_token(&t, TokenIdLBrace);
248 end_token(&t);
249 break;
250 case '}':
251 begin_token(&t, TokenIdRBrace);
252 end_token(&t);
253 break;
254 case '"':
255 begin_token(&t, TokenIdStringLiteral);
256 t.state = TokenizeStateString;
257 break;
258 case ';':
259 begin_token(&t, TokenIdSemicolon);
260 end_token(&t);
261 break;
262 case '+':
263 begin_token(&t, TokenIdPlus);
264 end_token(&t);
265 break;
266 default:
267 tokenize_error(&t, "invalid character: '%c'", c);
268 }
269 break;
270 case TokenizeStateDirective:
271 if (c == '\n') {
272 assert(t.cur_tok);
273 t.cur_tok->end_pos = t.pos;
274 t.cur_tok = nullptr;
275 t.state = TokenizeStateStart;
276 }
277 break;
278 case TokenizeStateSymbol:
279 switch (c) {
280 case ALPHA:
281 case DIGIT:
282 case '_':
283 break;
284 default:
285 put_back(&t, 1);
286 end_token(&t);
287 t.state = TokenizeStateStart;
288 break;
289 }
290 break;
291 case TokenizeStateString:
292 switch (c) {
293 case '"':
294 end_token(&t);
295 t.state = TokenizeStateStart;
296 break;
297 default:
298 break;
299 }
300 break;
301 case TokenizeStateNumber:
302 switch (c) {
303 case DIGIT:
304 break;
305 default:
306 put_back(&t, 1);
307 end_token(&t);
308 t.state = TokenizeStateStart;
309 break;
310 }
311 break;
312 }
313 if (c == '\n') {
314 t.line += 1;
315 t.column = 0;
316 } else {
317 t.column += 1;
318 }
319 }
320 return t.tokens;
321}
322
323static const char * token_name(Token *token) {
324 switch (token->id) {
325 case TokenIdDirective: return "Directive";
326 case TokenIdSymbol: return "Symbol";
327 case TokenIdLParen: return "LParen";
328 case TokenIdRParen: return "RParen";
329 case TokenIdComma: return "Comma";
330 case TokenIdStar: return "Star";
331 case TokenIdLBrace: return "LBrace";
332 case TokenIdRBrace: return "RBrace";
333 case TokenIdStringLiteral: return "StringLiteral";
334 case TokenIdSemicolon: return "Semicolon";
335 case TokenIdNumberLiteral: return "NumberLiteral";
336 case TokenIdPlus: return "Plus";
337 }
338 return "(invalid token)";
339}
340
341static void print_tokens(Buf *buf, ZigList<Token> *tokens) {
342 for (int i = 0; i < tokens->length; i += 1) {
343 Token *token = &tokens->at(i);
344 printf("%s ", token_name(token));
345 fwrite(buf->ptr + token->start_pos, 1, token->end_pos - token->start_pos, stdout);
346 printf("\n");
347 }
348}
349
59350int main(int argc, char **argv) {
60351 char *arg0 = argv[0];
61352 char *in_file = NULL;
......@@ -99,7 +390,9 @@ int main(int argc, char **argv) {
99390
100391 fprintf(stderr, "%s\n", in_data->ptr);
101392
102 //tokenize(in_data);
393 ZigList<Token> *tokens = tokenize(in_data);
394
395 print_tokens(in_data, tokens);
103396
104397
105398 return EXIT_SUCCESS;
src/util.hpp+15
......@@ -48,4 +48,19 @@ template <typename T, long n>
4848constexpr long array_length(const T (&)[n]) {
4949 return n;
5050}
51
52template <typename T>
53static inline T max(T a, T b) {
54 return (a >= b) ? a : b;
55}
56
57template <typename T>
58static inline T min(T a, T b) {
59 return (a <= b) ? a : b;
60}
61
62template<typename T>
63static inline T clamp(T min_value, T value, T max_value) {
64 return max(min(value, max_value), min_value);
65}
5166#endif