authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-08 23:56:07-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-08 23:56:07-05:00
logf466e539ef27d1cf90e2b163d5afbcf0bffc0aa5
tree7a85e5c73900d8044363ed946cf941c49937957c
parentd431b0fb99342f1e2629f5dd8bf1ca578a5e5d9a

tokenizing libc hello world

1 files changed, 90 insertions(+), 0 deletions(-)

src-self-hosted/main.zig+90
...@@ -90,6 +90,10 @@ const Token = struct {...@@ -90,6 +90,10 @@ const Token = struct {
90 Arrow,90 Arrow,
91 Colon,91 Colon,
92 Slash,92 Slash,
93 Comma,
94 Ampersand,
95 AmpersandEqual,
96 NumberLiteral,
93 Keyword_align,97 Keyword_align,
94 Keyword_and,98 Keyword_and,
95 Keyword_asm,99 Keyword_asm,
...@@ -189,6 +193,13 @@ const Tokenizer = struct {...@@ -189,6 +193,13 @@ const Tokenizer = struct {
189 Minus,193 Minus,
190 Slash,194 Slash,
191 LineComment,195 LineComment,
196 Zero,
197 NumberLiteral,
198 NumberDot,
199 FloatFraction,
200 FloatExponentUnsigned,
201 FloatExponentNumber,
202 Ampersand,
192 };203 };
193204
194 pub fn next(self: &Tokenizer) -> Token {205 pub fn next(self: &Tokenizer) -> Token {
...@@ -241,6 +252,11 @@ const Tokenizer = struct {...@@ -241,6 +252,11 @@ const Tokenizer = struct {
241 self.index += 1;252 self.index += 1;
242 break;253 break;
243 },254 },
255 ',' => {
256 result.id = Token.Id.Comma;
257 self.index += 1;
258 break;
259 },
244 ':' => {260 ':' => {
245 result.id = Token.Id.Colon;261 result.id = Token.Id.Colon;
246 self.index += 1;262 self.index += 1;
...@@ -272,12 +288,34 @@ const Tokenizer = struct {...@@ -272,12 +288,34 @@ const Tokenizer = struct {
272 '/' => {288 '/' => {
273 state = State.Slash;289 state = State.Slash;
274 },290 },
291 '&' => {
292 state = State.Ampersand;
293 },
294 '0' => {
295 state = State.Zero;
296 result.id = Token.Id.NumberLiteral;
297 },
298 '1'...'9' => {
299 state = State.NumberLiteral;
300 result.id = Token.Id.NumberLiteral;
301 },
275 else => {302 else => {
276 result.id = Token.Id.Invalid;303 result.id = Token.Id.Invalid;
277 self.index += 1;304 self.index += 1;
278 break;305 break;
279 },306 },
280 },307 },
308 State.Ampersand => switch (c) {
309 '=' => {
310 result.id = Token.Id.AmpersandEqual;
311 self.index += 1;
312 break;
313 },
314 else => {
315 result.id = Token.Id.Ampersand;
316 break;
317 },
318 },
281 State.Identifier => switch (c) {319 State.Identifier => switch (c) {
282 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},320 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
283 else => {321 else => {
...@@ -353,6 +391,58 @@ const Tokenizer = struct {...@@ -353,6 +391,58 @@ const Tokenizer = struct {
353 },391 },
354 else => {},392 else => {},
355 },393 },
394 State.Zero => switch (c) {
395 'b', 'o', 'x' => {
396 state = State.NumberLiteral;
397 },
398 else => {
399 // reinterpret as a normal number
400 self.index -= 1;
401 state = State.NumberLiteral;
402 },
403 },
404 State.NumberLiteral => switch (c) {
405 '.' => {
406 state = State.NumberDot;
407 },
408 'p', 'P', 'e', 'E' => {
409 state = State.FloatExponentUnsigned;
410 },
411 '0'...'9', 'a'...'f', 'A'...'F' => {},
412 else => break,
413 },
414 State.NumberDot => switch (c) {
415 '.' => {
416 self.index -= 1;
417 state = State.Start;
418 break;
419 },
420 else => {
421 self.index -= 1;
422 state = State.FloatFraction;
423 },
424 },
425 State.FloatFraction => switch (c) {
426 'p', 'P', 'e', 'E' => {
427 state = State.FloatExponentUnsigned;
428 },
429 '0'...'9', 'a'...'f', 'A'...'F' => {},
430 else => break,
431 },
432 State.FloatExponentUnsigned => switch (c) {
433 '+', '-' => {
434 state = State.FloatExponentNumber;
435 },
436 else => {
437 // reinterpret as a normal exponent number
438 self.index -= 1;
439 state = State.FloatExponentNumber;
440 }
441 },
442 State.FloatExponentNumber => switch (c) {
443 '0'...'9', 'a'...'f', 'A'...'F' => {},
444 else => break,
445 },
356 }446 }
357 }447 }
358 result.end = self.index;448 result.end = self.index;