authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-08 23:56:07-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-08 23:56:07-05:00
logf466e539ef27d1cf90e2b163d5afbcf0bffc0aa5
tree7a85e5c73900d8044363ed946cf941c49937957c
parentd431b0fb99342f1e2629f5dd8bf1ca578a5e5d9a

tokenizing libc hello world

1 files changed, 90 insertions(+), 0 deletions(-)

src-self-hosted/main.zig+90
......@@ -90,6 +90,10 @@ const Token = struct {
9090 Arrow,
9191 Colon,
9292 Slash,
93 Comma,
94 Ampersand,
95 AmpersandEqual,
96 NumberLiteral,
9397 Keyword_align,
9498 Keyword_and,
9599 Keyword_asm,
......@@ -189,6 +193,13 @@ const Tokenizer = struct {
189193 Minus,
190194 Slash,
191195 LineComment,
196 Zero,
197 NumberLiteral,
198 NumberDot,
199 FloatFraction,
200 FloatExponentUnsigned,
201 FloatExponentNumber,
202 Ampersand,
192203 };
193204
194205 pub fn next(self: &Tokenizer) -> Token {
......@@ -241,6 +252,11 @@ const Tokenizer = struct {
241252 self.index += 1;
242253 break;
243254 },
255 ',' => {
256 result.id = Token.Id.Comma;
257 self.index += 1;
258 break;
259 },
244260 ':' => {
245261 result.id = Token.Id.Colon;
246262 self.index += 1;
......@@ -272,12 +288,34 @@ const Tokenizer = struct {
272288 '/' => {
273289 state = State.Slash;
274290 },
291 '&' => {
292 state = State.Ampersand;
293 },
294 '0' => {
295 state = State.Zero;
296 result.id = Token.Id.NumberLiteral;
297 },
298 '1'...'9' => {
299 state = State.NumberLiteral;
300 result.id = Token.Id.NumberLiteral;
301 },
275302 else => {
276303 result.id = Token.Id.Invalid;
277304 self.index += 1;
278305 break;
279306 },
280307 },
308 State.Ampersand => switch (c) {
309 '=' => {
310 result.id = Token.Id.AmpersandEqual;
311 self.index += 1;
312 break;
313 },
314 else => {
315 result.id = Token.Id.Ampersand;
316 break;
317 },
318 },
281319 State.Identifier => switch (c) {
282320 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
283321 else => {
......@@ -353,6 +391,58 @@ const Tokenizer = struct {
353391 },
354392 else => {},
355393 },
394 State.Zero => switch (c) {
395 'b', 'o', 'x' => {
396 state = State.NumberLiteral;
397 },
398 else => {
399 // reinterpret as a normal number
400 self.index -= 1;
401 state = State.NumberLiteral;
402 },
403 },
404 State.NumberLiteral => switch (c) {
405 '.' => {
406 state = State.NumberDot;
407 },
408 'p', 'P', 'e', 'E' => {
409 state = State.FloatExponentUnsigned;
410 },
411 '0'...'9', 'a'...'f', 'A'...'F' => {},
412 else => break,
413 },
414 State.NumberDot => switch (c) {
415 '.' => {
416 self.index -= 1;
417 state = State.Start;
418 break;
419 },
420 else => {
421 self.index -= 1;
422 state = State.FloatFraction;
423 },
424 },
425 State.FloatFraction => switch (c) {
426 'p', 'P', 'e', 'E' => {
427 state = State.FloatExponentUnsigned;
428 },
429 '0'...'9', 'a'...'f', 'A'...'F' => {},
430 else => break,
431 },
432 State.FloatExponentUnsigned => switch (c) {
433 '+', '-' => {
434 state = State.FloatExponentNumber;
435 },
436 else => {
437 // reinterpret as a normal exponent number
438 self.index -= 1;
439 state = State.FloatExponentNumber;
440 }
441 },
442 State.FloatExponentNumber => switch (c) {
443 '0'...'9', 'a'...'f', 'A'...'F' => {},
444 else => break,
445 },
356446 }
357447 }
358448 result.end = self.index;