authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-05 17:44:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2015-08-05 17:44:05-07:00
loge09932928ac16681d71127f3d5cc0c488f2fdcc5
tree57ded4c21186e13a5e0283a6e5fa30bbe07738f6
parent899c9fe94e04f50c6176644b61e2d89cb74a8886

tokenize

4 files changed, 318 insertions(+), 13 deletions(-)

README.md+4
...@@ -1 +1,5 @@...@@ -1 +1,5 @@
1# zig lang1# zig lang
2
3C upgrade.
4
5Start with C.
src/list.hpp+5-12
...@@ -5,24 +5,21 @@...@@ -5,24 +5,21 @@
5 * See http://opensource.org/licenses/MIT5 * See http://opensource.org/licenses/MIT
6 */6 */
77
8#ifndef GROOVE_LIST_HPP8#ifndef ZIG_LIST_HPP
9#define GROOVE_LIST_HPP9#define ZIG_LIST_HPP
1010
11#include "util.hpp"11#include "util.hpp"
1212
13#include <assert.h>13#include <assert.h>
1414
15template<typename T>15template<typename T>
16struct GrooveList {16struct ZigList {
17 void deinit() {17 void deinit() {
18 deallocate(items);18 deallocate(items);
19 }19 }
20 void append(T item) {20 void append(T item) {
21 int err = ensure_capacity(length + 1);21 ensure_capacity(length + 1);
22 if (err)
23 return err;
24 items[length++] = item;22 items[length++] = item;
25 return 0;
26 }23 }
27 // remember that the pointer to this item is invalid after you24 // remember that the pointer to this item is invalid after you
28 // modify the length of the list25 // modify the length of the list
...@@ -57,11 +54,8 @@ struct GrooveList {...@@ -57,11 +54,8 @@ struct GrooveList {
5754
58 void resize(int new_length) {55 void resize(int new_length) {
59 assert(new_length >= 0);56 assert(new_length >= 0);
60 int err = ensure_capacity(new_length);57 ensure_capacity(new_length);
61 if (err)
62 return err;
63 length = new_length;58 length = new_length;
64 return 0;
65 }59 }
6660
67 void clear() {61 void clear() {
...@@ -76,7 +70,6 @@ struct GrooveList {...@@ -76,7 +70,6 @@ struct GrooveList {
76 items = reallocate_nonzero(items, better_capacity);70 items = reallocate_nonzero(items, better_capacity);
77 capacity = better_capacity;71 capacity = better_capacity;
78 }72 }
79 return 0;
80 }73 }
8174
82 T * items;75 T * items;
src/main.cpp+294-1
...@@ -7,6 +7,7 @@...@@ -7,6 +7,7 @@
77
8#include "config.h"8#include "config.h"
9#include "util.hpp"9#include "util.hpp"
10#include "list.hpp"
10#include <stdio.h>11#include <stdio.h>
11#include <string.h>12#include <string.h>
12#include <stdlib.h>13#include <stdlib.h>
...@@ -30,6 +31,13 @@ static Buf *alloc_buf(int size) {...@@ -30,6 +31,13 @@ static Buf *alloc_buf(int size) {
30 return buf;31 return buf;
31}32}
3233
34/*
35static void fprint_buf(FILE *f, Buf *buf) {
36 if (fwrite(buf->ptr, 1, buf->len, f))
37 zig_panic("error writing: %s", strerror(errno));
38}
39*/
40
33static int usage(char *arg0) {41static int usage(char *arg0) {
34 fprintf(stderr, "Usage: %s --output outfile code.zig\n"42 fprintf(stderr, "Usage: %s --output outfile code.zig\n"
35 "Other options:\n"43 "Other options:\n"
...@@ -56,6 +64,289 @@ static struct Buf *fetch_file(FILE *f) {...@@ -56,6 +64,289 @@ static struct Buf *fetch_file(FILE *f) {
56 return buf;64 return buf;
57}65}
5866
67#define WHITESPACE \
68 ' ': \
69 case '\t': \
70 case '\n': \
71 case '\f': \
72 case '\r': \
73 case 0xb
74
75#define DIGIT \
76 '0': \
77 case '1': \
78 case '2': \
79 case '3': \
80 case '4': \
81 case '5': \
82 case '6': \
83 case '7': \
84 case '8': \
85 case '9'
86
87#define ALPHA \
88 'a': \
89 case 'b': \
90 case 'c': \
91 case 'd': \
92 case 'e': \
93 case 'f': \
94 case 'g': \
95 case 'h': \
96 case 'i': \
97 case 'j': \
98 case 'k': \
99 case 'l': \
100 case 'm': \
101 case 'n': \
102 case 'o': \
103 case 'p': \
104 case 'q': \
105 case 'r': \
106 case 's': \
107 case 't': \
108 case 'u': \
109 case 'v': \
110 case 'w': \
111 case 'x': \
112 case 'y': \
113 case 'z': \
114 case 'A': \
115 case 'B': \
116 case 'C': \
117 case 'D': \
118 case 'E': \
119 case 'F': \
120 case 'G': \
121 case 'H': \
122 case 'I': \
123 case 'J': \
124 case 'K': \
125 case 'L': \
126 case 'M': \
127 case 'N': \
128 case 'O': \
129 case 'P': \
130 case 'Q': \
131 case 'R': \
132 case 'S': \
133 case 'T': \
134 case 'U': \
135 case 'V': \
136 case 'W': \
137 case 'X': \
138 case 'Y': \
139 case 'Z'
140
141enum TokenId {
142 TokenIdDirective,
143 TokenIdSymbol,
144 TokenIdLParen,
145 TokenIdRParen,
146 TokenIdComma,
147 TokenIdStar,
148 TokenIdLBrace,
149 TokenIdRBrace,
150 TokenIdStringLiteral,
151 TokenIdSemicolon,
152 TokenIdNumberLiteral,
153 TokenIdPlus,
154};
155
156struct Token {
157 TokenId id;
158 int start_pos;
159 int end_pos;
160};
161
162enum TokenizeState {
163 TokenizeStateStart,
164 TokenizeStateDirective,
165 TokenizeStateSymbol,
166 TokenizeStateString,
167 TokenizeStateNumber,
168};
169
170struct Tokenize {
171 int pos;
172 TokenizeState state;
173 ZigList<Token> *tokens;
174 int line;
175 int column;
176 Token *cur_tok;
177};
178
179__attribute__ ((format (printf, 2, 3)))
180static void tokenize_error(Tokenize *t, const char *format, ...) {
181 va_list ap;
182 va_start(ap, format);
183 fprintf(stderr, "Error. Line %d, column %d: ", t->line + 1, t->column + 1);
184 vfprintf(stderr, format, ap);
185 va_end(ap);
186 exit(EXIT_FAILURE);
187}
188
189static void begin_token(Tokenize *t, TokenId id) {
190 assert(!t->cur_tok);
191 t->tokens->add_one();
192 Token *token = &t->tokens->last();
193 token->id = id;
194 token->start_pos = t->pos;
195 t->cur_tok = token;
196}
197
198static void end_token(Tokenize *t) {
199 assert(t->cur_tok);
200 t->cur_tok->end_pos = t->pos + 1;
201 t->cur_tok = nullptr;
202}
203
204static void put_back(Tokenize *t, int count) {
205 t->pos -= count;
206}
207
208static ZigList<Token> *tokenize(Buf *buf) {
209 Tokenize t = {0};
210 t.tokens = allocate<ZigList<Token>>(1);
211 for (t.pos = 0; t.pos < buf->len; t.pos += 1) {
212 uint8_t c = buf->ptr[t.pos];
213 switch (t.state) {
214 case TokenizeStateStart:
215 switch (c) {
216 case WHITESPACE:
217 break;
218 case ALPHA:
219 t.state = TokenizeStateSymbol;
220 begin_token(&t, TokenIdSymbol);
221 break;
222 case DIGIT:
223 t.state = TokenizeStateNumber;
224 begin_token(&t, TokenIdNumberLiteral);
225 break;
226 case '#':
227 t.state = TokenizeStateDirective;
228 begin_token(&t, TokenIdDirective);
229 break;
230 case '(':
231 begin_token(&t, TokenIdLParen);
232 end_token(&t);
233 break;
234 case ')':
235 begin_token(&t, TokenIdLParen);
236 end_token(&t);
237 break;
238 case ',':
239 begin_token(&t, TokenIdComma);
240 end_token(&t);
241 break;
242 case '*':
243 begin_token(&t, TokenIdStar);
244 end_token(&t);
245 break;
246 case '{':
247 begin_token(&t, TokenIdLBrace);
248 end_token(&t);
249 break;
250 case '}':
251 begin_token(&t, TokenIdRBrace);
252 end_token(&t);
253 break;
254 case '"':
255 begin_token(&t, TokenIdStringLiteral);
256 t.state = TokenizeStateString;
257 break;
258 case ';':
259 begin_token(&t, TokenIdSemicolon);
260 end_token(&t);
261 break;
262 case '+':
263 begin_token(&t, TokenIdPlus);
264 end_token(&t);
265 break;
266 default:
267 tokenize_error(&t, "invalid character: '%c'", c);
268 }
269 break;
270 case TokenizeStateDirective:
271 if (c == '\n') {
272 assert(t.cur_tok);
273 t.cur_tok->end_pos = t.pos;
274 t.cur_tok = nullptr;
275 t.state = TokenizeStateStart;
276 }
277 break;
278 case TokenizeStateSymbol:
279 switch (c) {
280 case ALPHA:
281 case DIGIT:
282 case '_':
283 break;
284 default:
285 put_back(&t, 1);
286 end_token(&t);
287 t.state = TokenizeStateStart;
288 break;
289 }
290 break;
291 case TokenizeStateString:
292 switch (c) {
293 case '"':
294 end_token(&t);
295 t.state = TokenizeStateStart;
296 break;
297 default:
298 break;
299 }
300 break;
301 case TokenizeStateNumber:
302 switch (c) {
303 case DIGIT:
304 break;
305 default:
306 put_back(&t, 1);
307 end_token(&t);
308 t.state = TokenizeStateStart;
309 break;
310 }
311 break;
312 }
313 if (c == '\n') {
314 t.line += 1;
315 t.column = 0;
316 } else {
317 t.column += 1;
318 }
319 }
320 return t.tokens;
321}
322
323static const char * token_name(Token *token) {
324 switch (token->id) {
325 case TokenIdDirective: return "Directive";
326 case TokenIdSymbol: return "Symbol";
327 case TokenIdLParen: return "LParen";
328 case TokenIdRParen: return "RParen";
329 case TokenIdComma: return "Comma";
330 case TokenIdStar: return "Star";
331 case TokenIdLBrace: return "LBrace";
332 case TokenIdRBrace: return "RBrace";
333 case TokenIdStringLiteral: return "StringLiteral";
334 case TokenIdSemicolon: return "Semicolon";
335 case TokenIdNumberLiteral: return "NumberLiteral";
336 case TokenIdPlus: return "Plus";
337 }
338 return "(invalid token)";
339}
340
341static void print_tokens(Buf *buf, ZigList<Token> *tokens) {
342 for (int i = 0; i < tokens->length; i += 1) {
343 Token *token = &tokens->at(i);
344 printf("%s ", token_name(token));
345 fwrite(buf->ptr + token->start_pos, 1, token->end_pos - token->start_pos, stdout);
346 printf("\n");
347 }
348}
349
59int main(int argc, char **argv) {350int main(int argc, char **argv) {
60 char *arg0 = argv[0];351 char *arg0 = argv[0];
61 char *in_file = NULL;352 char *in_file = NULL;
...@@ -99,7 +390,9 @@ int main(int argc, char **argv) {...@@ -99,7 +390,9 @@ int main(int argc, char **argv) {
99390
100 fprintf(stderr, "%s\n", in_data->ptr);391 fprintf(stderr, "%s\n", in_data->ptr);
101392
102 //tokenize(in_data);393 ZigList<Token> *tokens = tokenize(in_data);
394
395 print_tokens(in_data, tokens);
103396
104397
105 return EXIT_SUCCESS;398 return EXIT_SUCCESS;
src/util.hpp+15
...@@ -48,4 +48,19 @@ template <typename T, long n>...@@ -48,4 +48,19 @@ template <typename T, long n>
48constexpr long array_length(const T (&)[n]) {48constexpr long array_length(const T (&)[n]) {
49 return n;49 return n;
50}50}
51
52template <typename T>
53static inline T max(T a, T b) {
54 return (a >= b) ? a : b;
55}
56
57template <typename T>
58static inline T min(T a, T b) {
59 return (a <= b) ? a : b;
60}
61
62template<typename T>
63static inline T clamp(T min_value, T value, T max_value) {
64 return max(min(value, max_value), min_value);
65}
51#endif66#endif