authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-12 18:17:27-07:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-12 18:17:27-07:00
logeb1542c1027481546d0efdf6a5faac2593b9e8e8
tree4191d6e86dd5ac7a60be0ce13c73f22624c74b25
parent208258820108b1e921d456e99d48844ebbe91040

tokenizing assignment operators

3 files changed, 227 insertions(+), 62 deletions(-)

doc/langref.md+4-2
......@@ -88,7 +88,9 @@ AsmInputItem : token(LBracket) token(Symbol) token(RBracket) token(String) token
8888
8989AsmClobbers: token(Colon) list(token(String), token(Comma))
9090
91AssignmentExpression : BoolOrExpression token(Equal) BoolOrExpression | BoolOrExpression
91AssignmentExpression : BoolOrExpression AssignmentOperator BoolOrExpression | BoolOrExpression
92
93AssignmentOperator : token(Eq) | token(TimesEq) | token(DivEq) | token(ModEq) | token(PlusEq) | token(MinusEq) | token(BitShiftLeftEq) | token(BitShiftRightEq) | token(BitAndEq) | token(BitXorEq) | token(BitOrEq) | token(BoolAndEq) | token(BoolOrEq)
9294
9395BlockExpression : IfExpression | Block
9496
......@@ -164,7 +166,7 @@ as
164166== != < > <= >=
165167&&
166168||
167= += -=
169= *= /= %= += -= <<= >>= &= ^= |= &&= ||=
168170```
169171
170172## Literals
src/tokenizer.cpp+211-60
......@@ -96,20 +96,28 @@ enum TokenizeState {
9696 TokenizeStateSymbolFirst,
9797 TokenizeStateNumber,
9898 TokenizeStateString,
99 TokenizeStateSawDash,
99 TokenizeStateSawStar,
100100 TokenizeStateSawSlash,
101 TokenizeStateSawPercent,
102 TokenizeStateSawPlus,
103 TokenizeStateSawDash,
104 TokenizeStateSawAmpersand,
105 TokenizeStateSawAmpersandAmpersand,
106 TokenizeStateSawCaret,
107 TokenizeStateSawPipe,
108 TokenizeStateSawPipePipe,
101109 TokenizeStateLineComment,
102110 TokenizeStateMultiLineComment,
103111 TokenizeStateMultiLineCommentSlash,
104112 TokenizeStateMultiLineCommentStar,
105 TokenizeStatePipe,
106 TokenizeStateAmpersand,
107 TokenizeStateEq,
108 TokenizeStateBang,
109 TokenizeStateLessThan,
110 TokenizeStateGreaterThan,
111 TokenizeStateDot,
112 TokenizeStateDotDot,
113 TokenizeStateSawEq,
114 TokenizeStateSawBang,
115 TokenizeStateSawLessThan,
116 TokenizeStateSawLessThanLessThan,
117 TokenizeStateSawGreaterThan,
118 TokenizeStateSawGreaterThanGreaterThan,
119 TokenizeStateSawDot,
120 TokenizeStateSawDotDot,
113121 TokenizeStateError,
114122};
115123
......@@ -259,14 +267,6 @@ void tokenize(Buf *buf, Tokenization *out) {
259267 begin_token(&t, TokenIdComma);
260268 end_token(&t);
261269 break;
262 case '*':
263 begin_token(&t, TokenIdStar);
264 end_token(&t);
265 break;
266 case '%':
267 begin_token(&t, TokenIdPercent);
268 end_token(&t);
269 break;
270270 case '{':
271271 begin_token(&t, TokenIdLBrace);
272272 end_token(&t);
......@@ -291,9 +291,25 @@ void tokenize(Buf *buf, Tokenization *out) {
291291 begin_token(&t, TokenIdColon);
292292 end_token(&t);
293293 break;
294 case '#':
295 begin_token(&t, TokenIdNumberSign);
296 end_token(&t);
297 break;
298 case '*':
299 begin_token(&t, TokenIdStar);
300 t.state = TokenizeStateSawStar;
301 break;
302 case '/':
303 begin_token(&t, TokenIdSlash);
304 t.state = TokenizeStateSawSlash;
305 break;
306 case '%':
307 begin_token(&t, TokenIdPercent);
308 t.state = TokenizeStateSawPercent;
309 break;
294310 case '+':
295311 begin_token(&t, TokenIdPlus);
296 end_token(&t);
312 t.state = TokenizeStateSawPlus;
297313 break;
298314 case '~':
299315 begin_token(&t, TokenIdTilde);
......@@ -303,54 +319,46 @@ void tokenize(Buf *buf, Tokenization *out) {
303319 begin_token(&t, TokenIdDash);
304320 t.state = TokenizeStateSawDash;
305321 break;
306 case '#':
307 begin_token(&t, TokenIdNumberSign);
308 end_token(&t);
322 case '&':
323 begin_token(&t, TokenIdBinAnd);
324 t.state = TokenizeStateSawAmpersand;
309325 break;
310326 case '^':
311327 begin_token(&t, TokenIdBinXor);
312 end_token(&t);
313 break;
314 case '/':
315 begin_token(&t, TokenIdSlash);
316 t.state = TokenizeStateSawSlash;
328 t.state = TokenizeStateSawCaret;
317329 break;
318330 case '|':
319331 begin_token(&t, TokenIdBinOr);
320 t.state = TokenizeStatePipe;
321 break;
322 case '&':
323 begin_token(&t, TokenIdBinAnd);
324 t.state = TokenizeStateAmpersand;
332 t.state = TokenizeStateSawPipe;
325333 break;
326334 case '=':
327335 begin_token(&t, TokenIdEq);
328 t.state = TokenizeStateEq;
336 t.state = TokenizeStateSawEq;
329337 break;
330338 case '!':
331339 begin_token(&t, TokenIdBang);
332 t.state = TokenizeStateBang;
340 t.state = TokenizeStateSawBang;
333341 break;
334342 case '<':
335343 begin_token(&t, TokenIdCmpLessThan);
336 t.state = TokenizeStateLessThan;
344 t.state = TokenizeStateSawLessThan;
337345 break;
338346 case '>':
339347 begin_token(&t, TokenIdCmpGreaterThan);
340 t.state = TokenizeStateGreaterThan;
348 t.state = TokenizeStateSawGreaterThan;
341349 break;
342350 case '.':
343351 begin_token(&t, TokenIdDot);
344 t.state = TokenizeStateDot;
352 t.state = TokenizeStateSawDot;
345353 break;
346354 default:
347355 tokenize_error(&t, "invalid character: '%c'", c);
348356 }
349357 break;
350 case TokenizeStateDot:
358 case TokenizeStateSawDot:
351359 switch (c) {
352360 case '.':
353 t.state = TokenizeStateDotDot;
361 t.state = TokenizeStateSawDotDot;
354362 t.cur_tok->id = TokenIdEllipsis;
355363 break;
356364 default:
......@@ -360,20 +368,17 @@ void tokenize(Buf *buf, Tokenization *out) {
360368 continue;
361369 }
362370 break;
363 case TokenizeStateDotDot:
371 case TokenizeStateSawDotDot:
364372 switch (c) {
365373 case '.':
366374 t.state = TokenizeStateStart;
367375 end_token(&t);
368376 break;
369377 default:
370 t.pos -= 1;
371 end_token(&t);
372 t.state = TokenizeStateStart;
373 continue;
378 tokenize_error(&t, "invalid character: '%c'", c);
374379 }
375380 break;
376 case TokenizeStateGreaterThan:
381 case TokenizeStateSawGreaterThan:
377382 switch (c) {
378383 case '=':
379384 t.cur_tok->id = TokenIdCmpGreaterOrEq;
......@@ -382,9 +387,21 @@ void tokenize(Buf *buf, Tokenization *out) {
382387 break;
383388 case '>':
384389 t.cur_tok->id = TokenIdBitShiftRight;
390 t.state = TokenizeStateSawGreaterThanGreaterThan;
391 break;
392 default:
393 t.pos -= 1;
394 end_token(&t);
395 t.state = TokenizeStateStart;
396 continue;
397 }
398 break;
399 case TokenizeStateSawGreaterThanGreaterThan:
400 switch (c) {
401 case '=':
402 t.cur_tok->id = TokenIdBitShiftRightEq;
385403 end_token(&t);
386404 t.state = TokenizeStateStart;
387 break;
388405 default:
389406 t.pos -= 1;
390407 end_token(&t);
......@@ -392,7 +409,7 @@ void tokenize(Buf *buf, Tokenization *out) {
392409 continue;
393410 }
394411 break;
395 case TokenizeStateLessThan:
412 case TokenizeStateSawLessThan:
396413 switch (c) {
397414 case '=':
398415 t.cur_tok->id = TokenIdCmpLessOrEq;
......@@ -400,9 +417,21 @@ void tokenize(Buf *buf, Tokenization *out) {
400417 t.state = TokenizeStateStart;
401418 case '<':
402419 t.cur_tok->id = TokenIdBitShiftLeft;
420 t.state = TokenizeStateSawLessThanLessThan;
421 break;
422 default:
423 t.pos -= 1;
424 end_token(&t);
425 t.state = TokenizeStateStart;
426 continue;
427 }
428 break;
429 case TokenizeStateSawLessThanLessThan:
430 switch (c) {
431 case '=':
432 t.cur_tok->id = TokenIdBitShiftLeftEq;
403433 end_token(&t);
404434 t.state = TokenizeStateStart;
405 break;
406435 default:
407436 t.pos -= 1;
408437 end_token(&t);
......@@ -410,7 +439,7 @@ void tokenize(Buf *buf, Tokenization *out) {
410439 continue;
411440 }
412441 break;
413 case TokenizeStateBang:
442 case TokenizeStateSawBang:
414443 switch (c) {
415444 case '=':
416445 t.cur_tok->id = TokenIdCmpNotEq;
......@@ -424,7 +453,7 @@ void tokenize(Buf *buf, Tokenization *out) {
424453 continue;
425454 }
426455 break;
427 case TokenizeStateEq:
456 case TokenizeStateSawEq:
428457 switch (c) {
429458 case '=':
430459 t.cur_tok->id = TokenIdCmpEq;
......@@ -438,10 +467,56 @@ void tokenize(Buf *buf, Tokenization *out) {
438467 continue;
439468 }
440469 break;
441 case TokenizeStateAmpersand:
470 case TokenizeStateSawStar:
471 switch (c) {
472 case '=':
473 t.cur_tok->id = TokenIdTimesEq;
474 end_token(&t);
475 t.state = TokenizeStateStart;
476 break;
477 default:
478 t.pos -= 1;
479 end_token(&t);
480 t.state = TokenizeStateStart;
481 continue;
482 }
483 break;
484 case TokenizeStateSawPercent:
485 switch (c) {
486 case '=':
487 t.cur_tok->id = TokenIdModEq;
488 end_token(&t);
489 t.state = TokenizeStateStart;
490 break;
491 default:
492 t.pos -= 1;
493 end_token(&t);
494 t.state = TokenizeStateStart;
495 continue;
496 }
497 break;
498 case TokenizeStateSawPlus:
499 switch (c) {
500 case '=':
501 t.cur_tok->id = TokenIdPlusEq;
502 end_token(&t);
503 t.state = TokenizeStateStart;
504 break;
505 default:
506 t.pos -= 1;
507 end_token(&t);
508 t.state = TokenizeStateStart;
509 continue;
510 }
511 break;
512 case TokenizeStateSawAmpersand:
442513 switch (c) {
443514 case '&':
444515 t.cur_tok->id = TokenIdBoolAnd;
516 t.state = TokenizeStateSawAmpersandAmpersand;
517 break;
518 case '=':
519 t.cur_tok->id = TokenIdBitAndEq;
445520 end_token(&t);
446521 t.state = TokenizeStateStart;
447522 break;
......@@ -452,10 +527,56 @@ void tokenize(Buf *buf, Tokenization *out) {
452527 continue;
453528 }
454529 break;
455 case TokenizeStatePipe:
530 case TokenizeStateSawAmpersandAmpersand:
531 switch (c) {
532 case '=':
533 t.cur_tok->id = TokenIdBoolAndEq;
534 end_token(&t);
535 t.state = TokenizeStateStart;
536 break;
537 default:
538 t.pos -= 1;
539 end_token(&t);
540 t.state = TokenizeStateStart;
541 continue;
542 }
543 break;
544 case TokenizeStateSawCaret:
545 switch (c) {
546 case '=':
547 t.cur_tok->id = TokenIdBitXorEq;
548 end_token(&t);
549 t.state = TokenizeStateStart;
550 break;
551 default:
552 t.pos -= 1;
553 end_token(&t);
554 t.state = TokenizeStateStart;
555 continue;
556 }
557 break;
558 case TokenizeStateSawPipe:
456559 switch (c) {
457560 case '|':
458561 t.cur_tok->id = TokenIdBoolOr;
562 t.state = TokenizeStateSawPipePipe;
563 break;
564 case '=':
565 t.cur_tok->id = TokenIdBitOrEq;
566 end_token(&t);
567 t.state = TokenizeStateStart;
568 break;
569 default:
570 t.pos -= 1;
571 end_token(&t);
572 t.state = TokenizeStateStart;
573 continue;
574 }
575 break;
576 case TokenizeStateSawPipePipe:
577 switch (c) {
578 case '=':
579 t.cur_tok->id = TokenIdBoolOrEq;
459580 end_token(&t);
460581 t.state = TokenizeStateStart;
461582 break;
......@@ -477,6 +598,11 @@ void tokenize(Buf *buf, Tokenization *out) {
477598 t.state = TokenizeStateMultiLineComment;
478599 t.multi_line_comment_count = 1;
479600 break;
601 case '=':
602 t.cur_tok->id = TokenIdDivEq;
603 end_token(&t);
604 t.state = TokenizeStateStart;
605 break;
480606 default:
481607 t.pos -= 1;
482608 end_token(&t);
......@@ -592,6 +718,11 @@ void tokenize(Buf *buf, Tokenization *out) {
592718 end_token(&t);
593719 t.state = TokenizeStateStart;
594720 break;
721 case '=':
722 t.cur_tok->id = TokenIdMinusEq;
723 end_token(&t);
724 t.state = TokenizeStateStart;
725 break;
595726 default:
596727 t.pos -= 1;
597728 end_token(&t);
......@@ -619,18 +750,26 @@ void tokenize(Buf *buf, Tokenization *out) {
619750 case TokenizeStateSymbol:
620751 case TokenizeStateSymbolFirst:
621752 case TokenizeStateNumber:
753 case TokenizeStateSawStar:
754 case TokenizeStateSawSlash:
755 case TokenizeStateSawPercent:
756 case TokenizeStateSawPlus:
622757 case TokenizeStateSawDash:
623 case TokenizeStatePipe:
624 case TokenizeStateAmpersand:
625 case TokenizeStateEq:
626 case TokenizeStateBang:
627 case TokenizeStateLessThan:
628 case TokenizeStateGreaterThan:
629 case TokenizeStateDot:
758 case TokenizeStateSawAmpersand:
759 case TokenizeStateSawAmpersandAmpersand:
760 case TokenizeStateSawCaret:
761 case TokenizeStateSawPipe:
762 case TokenizeStateSawPipePipe:
763 case TokenizeStateSawEq:
764 case TokenizeStateSawBang:
765 case TokenizeStateSawLessThan:
766 case TokenizeStateSawLessThanLessThan:
767 case TokenizeStateSawGreaterThan:
768 case TokenizeStateSawGreaterThanGreaterThan:
769 case TokenizeStateSawDot:
630770 end_token(&t);
631771 break;
632 case TokenizeStateSawSlash:
633 case TokenizeStateDotDot:
772 case TokenizeStateSawDotDot:
634773 tokenize_error(&t, "unexpected EOF");
635774 break;
636775 case TokenizeStateLineComment:
......@@ -695,6 +834,18 @@ static const char * token_name(Token *token) {
695834 case TokenIdBoolOr: return "BoolOr";
696835 case TokenIdBoolAnd: return "BoolAnd";
697836 case TokenIdEq: return "Eq";
837 case TokenIdTimesEq: return "TimesEq";
838 case TokenIdDivEq: return "DivEq";
839 case TokenIdModEq: return "ModEq";
840 case TokenIdPlusEq: return "PlusEq";
841 case TokenIdMinusEq: return "MinusEq";
842 case TokenIdBitShiftLeftEq: return "BitShiftLeftEq";
843 case TokenIdBitShiftRightEq: return "BitShiftRightEq";
844 case TokenIdBitAndEq: return "BitAndEq";
845 case TokenIdBitXorEq: return "BitXorEq";
846 case TokenIdBitOrEq: return "BitOrEq";
847 case TokenIdBoolAndEq: return "BoolAndEq";
848 case TokenIdBoolOrEq: return "BoolOrEq";
698849 case TokenIdBang: return "Bang";
699850 case TokenIdTilde: return "Tilde";
700851 case TokenIdCmpEq: return "CmpEq";
src/tokenizer.hpp+12
......@@ -55,6 +55,18 @@ enum TokenId {
5555 TokenIdBinAnd,
5656 TokenIdBinXor,
5757 TokenIdEq,
58 TokenIdTimesEq,
59 TokenIdDivEq,
60 TokenIdModEq,
61 TokenIdPlusEq,
62 TokenIdMinusEq,
63 TokenIdBitShiftLeftEq,
64 TokenIdBitShiftRightEq,
65 TokenIdBitAndEq,
66 TokenIdBitXorEq,
67 TokenIdBitOrEq,
68 TokenIdBoolAndEq,
69 TokenIdBoolOrEq,
5870 TokenIdCmpEq,
5971 TokenIdBang,
6072 TokenIdTilde,