authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-12 18:17:27-07:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-12 18:17:27-07:00
logeb1542c1027481546d0efdf6a5faac2593b9e8e8
tree4191d6e86dd5ac7a60be0ce13c73f22624c74b25
parent208258820108b1e921d456e99d48844ebbe91040

tokenizing assignment operators

3 files changed, 227 insertions(+), 62 deletions(-)

doc/langref.md+4-2
...@@ -88,7 +88,9 @@ AsmInputItem : token(LBracket) token(Symbol) token(RBracket) token(String) token...@@ -88,7 +88,9 @@ AsmInputItem : token(LBracket) token(Symbol) token(RBracket) token(String) token
8888
89AsmClobbers: token(Colon) list(token(String), token(Comma))89AsmClobbers: token(Colon) list(token(String), token(Comma))
9090
91AssignmentExpression : BoolOrExpression token(Equal) BoolOrExpression | BoolOrExpression91AssignmentExpression : BoolOrExpression AssignmentOperator BoolOrExpression | BoolOrExpression
92
93AssignmentOperator : token(Eq) | token(TimesEq) | token(DivEq) | token(ModEq) | token(PlusEq) | token(MinusEq) | token(BitShiftLeftEq) | token(BitShiftRightEq) | token(BitAndEq) | token(BitXorEq) | token(BitOrEq) | token(BoolAndEq) | token(BoolOrEq)
9294
93BlockExpression : IfExpression | Block95BlockExpression : IfExpression | Block
9496
...@@ -164,7 +166,7 @@ as...@@ -164,7 +166,7 @@ as
164== != < > <= >=166== != < > <= >=
165&&167&&
166||168||
167= += -=169= *= /= %= += -= <<= >>= &= ^= |= &&= ||=
168```170```
169171
170## Literals172## Literals
src/tokenizer.cpp+211-60
...@@ -96,20 +96,28 @@ enum TokenizeState {...@@ -96,20 +96,28 @@ enum TokenizeState {
96 TokenizeStateSymbolFirst,96 TokenizeStateSymbolFirst,
97 TokenizeStateNumber,97 TokenizeStateNumber,
98 TokenizeStateString,98 TokenizeStateString,
99 TokenizeStateSawDash,99 TokenizeStateSawStar,
100 TokenizeStateSawSlash,100 TokenizeStateSawSlash,
101 TokenizeStateSawPercent,
102 TokenizeStateSawPlus,
103 TokenizeStateSawDash,
104 TokenizeStateSawAmpersand,
105 TokenizeStateSawAmpersandAmpersand,
106 TokenizeStateSawCaret,
107 TokenizeStateSawPipe,
108 TokenizeStateSawPipePipe,
101 TokenizeStateLineComment,109 TokenizeStateLineComment,
102 TokenizeStateMultiLineComment,110 TokenizeStateMultiLineComment,
103 TokenizeStateMultiLineCommentSlash,111 TokenizeStateMultiLineCommentSlash,
104 TokenizeStateMultiLineCommentStar,112 TokenizeStateMultiLineCommentStar,
105 TokenizeStatePipe,113 TokenizeStateSawEq,
106 TokenizeStateAmpersand,114 TokenizeStateSawBang,
107 TokenizeStateEq,115 TokenizeStateSawLessThan,
108 TokenizeStateBang,116 TokenizeStateSawLessThanLessThan,
109 TokenizeStateLessThan,117 TokenizeStateSawGreaterThan,
110 TokenizeStateGreaterThan,118 TokenizeStateSawGreaterThanGreaterThan,
111 TokenizeStateDot,119 TokenizeStateSawDot,
112 TokenizeStateDotDot,120 TokenizeStateSawDotDot,
113 TokenizeStateError,121 TokenizeStateError,
114};122};
115123
...@@ -259,14 +267,6 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -259,14 +267,6 @@ void tokenize(Buf *buf, Tokenization *out) {
259 begin_token(&t, TokenIdComma);267 begin_token(&t, TokenIdComma);
260 end_token(&t);268 end_token(&t);
261 break;269 break;
262 case '*':
263 begin_token(&t, TokenIdStar);
264 end_token(&t);
265 break;
266 case '%':
267 begin_token(&t, TokenIdPercent);
268 end_token(&t);
269 break;
270 case '{':270 case '{':
271 begin_token(&t, TokenIdLBrace);271 begin_token(&t, TokenIdLBrace);
272 end_token(&t);272 end_token(&t);
...@@ -291,9 +291,25 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -291,9 +291,25 @@ void tokenize(Buf *buf, Tokenization *out) {
291 begin_token(&t, TokenIdColon);291 begin_token(&t, TokenIdColon);
292 end_token(&t);292 end_token(&t);
293 break;293 break;
294 case '#':
295 begin_token(&t, TokenIdNumberSign);
296 end_token(&t);
297 break;
298 case '*':
299 begin_token(&t, TokenIdStar);
300 t.state = TokenizeStateSawStar;
301 break;
302 case '/':
303 begin_token(&t, TokenIdSlash);
304 t.state = TokenizeStateSawSlash;
305 break;
306 case '%':
307 begin_token(&t, TokenIdPercent);
308 t.state = TokenizeStateSawPercent;
309 break;
294 case '+':310 case '+':
295 begin_token(&t, TokenIdPlus);311 begin_token(&t, TokenIdPlus);
296 end_token(&t);312 t.state = TokenizeStateSawPlus;
297 break;313 break;
298 case '~':314 case '~':
299 begin_token(&t, TokenIdTilde);315 begin_token(&t, TokenIdTilde);
...@@ -303,54 +319,46 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -303,54 +319,46 @@ void tokenize(Buf *buf, Tokenization *out) {
303 begin_token(&t, TokenIdDash);319 begin_token(&t, TokenIdDash);
304 t.state = TokenizeStateSawDash;320 t.state = TokenizeStateSawDash;
305 break;321 break;
306 case '#':322 case '&':
307 begin_token(&t, TokenIdNumberSign);323 begin_token(&t, TokenIdBinAnd);
308 end_token(&t);324 t.state = TokenizeStateSawAmpersand;
309 break;325 break;
310 case '^':326 case '^':
311 begin_token(&t, TokenIdBinXor);327 begin_token(&t, TokenIdBinXor);
312 end_token(&t);328 t.state = TokenizeStateSawCaret;
313 break;
314 case '/':
315 begin_token(&t, TokenIdSlash);
316 t.state = TokenizeStateSawSlash;
317 break;329 break;
318 case '|':330 case '|':
319 begin_token(&t, TokenIdBinOr);331 begin_token(&t, TokenIdBinOr);
320 t.state = TokenizeStatePipe;332 t.state = TokenizeStateSawPipe;
321 break;
322 case '&':
323 begin_token(&t, TokenIdBinAnd);
324 t.state = TokenizeStateAmpersand;
325 break;333 break;
326 case '=':334 case '=':
327 begin_token(&t, TokenIdEq);335 begin_token(&t, TokenIdEq);
328 t.state = TokenizeStateEq;336 t.state = TokenizeStateSawEq;
329 break;337 break;
330 case '!':338 case '!':
331 begin_token(&t, TokenIdBang);339 begin_token(&t, TokenIdBang);
332 t.state = TokenizeStateBang;340 t.state = TokenizeStateSawBang;
333 break;341 break;
334 case '<':342 case '<':
335 begin_token(&t, TokenIdCmpLessThan);343 begin_token(&t, TokenIdCmpLessThan);
336 t.state = TokenizeStateLessThan;344 t.state = TokenizeStateSawLessThan;
337 break;345 break;
338 case '>':346 case '>':
339 begin_token(&t, TokenIdCmpGreaterThan);347 begin_token(&t, TokenIdCmpGreaterThan);
340 t.state = TokenizeStateGreaterThan;348 t.state = TokenizeStateSawGreaterThan;
341 break;349 break;
342 case '.':350 case '.':
343 begin_token(&t, TokenIdDot);351 begin_token(&t, TokenIdDot);
344 t.state = TokenizeStateDot;352 t.state = TokenizeStateSawDot;
345 break;353 break;
346 default:354 default:
347 tokenize_error(&t, "invalid character: '%c'", c);355 tokenize_error(&t, "invalid character: '%c'", c);
348 }356 }
349 break;357 break;
350 case TokenizeStateDot:358 case TokenizeStateSawDot:
351 switch (c) {359 switch (c) {
352 case '.':360 case '.':
353 t.state = TokenizeStateDotDot;361 t.state = TokenizeStateSawDotDot;
354 t.cur_tok->id = TokenIdEllipsis;362 t.cur_tok->id = TokenIdEllipsis;
355 break;363 break;
356 default:364 default:
...@@ -360,20 +368,17 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -360,20 +368,17 @@ void tokenize(Buf *buf, Tokenization *out) {
360 continue;368 continue;
361 }369 }
362 break;370 break;
363 case TokenizeStateDotDot:371 case TokenizeStateSawDotDot:
364 switch (c) {372 switch (c) {
365 case '.':373 case '.':
366 t.state = TokenizeStateStart;374 t.state = TokenizeStateStart;
367 end_token(&t);375 end_token(&t);
368 break;376 break;
369 default:377 default:
370 t.pos -= 1;378 tokenize_error(&t, "invalid character: '%c'", c);
371 end_token(&t);
372 t.state = TokenizeStateStart;
373 continue;
374 }379 }
375 break;380 break;
376 case TokenizeStateGreaterThan:381 case TokenizeStateSawGreaterThan:
377 switch (c) {382 switch (c) {
378 case '=':383 case '=':
379 t.cur_tok->id = TokenIdCmpGreaterOrEq;384 t.cur_tok->id = TokenIdCmpGreaterOrEq;
...@@ -382,9 +387,21 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -382,9 +387,21 @@ void tokenize(Buf *buf, Tokenization *out) {
382 break;387 break;
383 case '>':388 case '>':
384 t.cur_tok->id = TokenIdBitShiftRight;389 t.cur_tok->id = TokenIdBitShiftRight;
390 t.state = TokenizeStateSawGreaterThanGreaterThan;
391 break;
392 default:
393 t.pos -= 1;
394 end_token(&t);
395 t.state = TokenizeStateStart;
396 continue;
397 }
398 break;
399 case TokenizeStateSawGreaterThanGreaterThan:
400 switch (c) {
401 case '=':
402 t.cur_tok->id = TokenIdBitShiftRightEq;
385 end_token(&t);403 end_token(&t);
386 t.state = TokenizeStateStart;404 t.state = TokenizeStateStart;
387 break;
388 default:405 default:
389 t.pos -= 1;406 t.pos -= 1;
390 end_token(&t);407 end_token(&t);
...@@ -392,7 +409,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -392,7 +409,7 @@ void tokenize(Buf *buf, Tokenization *out) {
392 continue;409 continue;
393 }410 }
394 break;411 break;
395 case TokenizeStateLessThan:412 case TokenizeStateSawLessThan:
396 switch (c) {413 switch (c) {
397 case '=':414 case '=':
398 t.cur_tok->id = TokenIdCmpLessOrEq;415 t.cur_tok->id = TokenIdCmpLessOrEq;
...@@ -400,9 +417,21 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -400,9 +417,21 @@ void tokenize(Buf *buf, Tokenization *out) {
400 t.state = TokenizeStateStart;417 t.state = TokenizeStateStart;
401 case '<':418 case '<':
402 t.cur_tok->id = TokenIdBitShiftLeft;419 t.cur_tok->id = TokenIdBitShiftLeft;
420 t.state = TokenizeStateSawLessThanLessThan;
421 break;
422 default:
423 t.pos -= 1;
424 end_token(&t);
425 t.state = TokenizeStateStart;
426 continue;
427 }
428 break;
429 case TokenizeStateSawLessThanLessThan:
430 switch (c) {
431 case '=':
432 t.cur_tok->id = TokenIdBitShiftLeftEq;
403 end_token(&t);433 end_token(&t);
404 t.state = TokenizeStateStart;434 t.state = TokenizeStateStart;
405 break;
406 default:435 default:
407 t.pos -= 1;436 t.pos -= 1;
408 end_token(&t);437 end_token(&t);
...@@ -410,7 +439,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -410,7 +439,7 @@ void tokenize(Buf *buf, Tokenization *out) {
410 continue;439 continue;
411 }440 }
412 break;441 break;
413 case TokenizeStateBang:442 case TokenizeStateSawBang:
414 switch (c) {443 switch (c) {
415 case '=':444 case '=':
416 t.cur_tok->id = TokenIdCmpNotEq;445 t.cur_tok->id = TokenIdCmpNotEq;
...@@ -424,7 +453,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -424,7 +453,7 @@ void tokenize(Buf *buf, Tokenization *out) {
424 continue;453 continue;
425 }454 }
426 break;455 break;
427 case TokenizeStateEq:456 case TokenizeStateSawEq:
428 switch (c) {457 switch (c) {
429 case '=':458 case '=':
430 t.cur_tok->id = TokenIdCmpEq;459 t.cur_tok->id = TokenIdCmpEq;
...@@ -438,10 +467,56 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -438,10 +467,56 @@ void tokenize(Buf *buf, Tokenization *out) {
438 continue;467 continue;
439 }468 }
440 break;469 break;
441 case TokenizeStateAmpersand:470 case TokenizeStateSawStar:
471 switch (c) {
472 case '=':
473 t.cur_tok->id = TokenIdTimesEq;
474 end_token(&t);
475 t.state = TokenizeStateStart;
476 break;
477 default:
478 t.pos -= 1;
479 end_token(&t);
480 t.state = TokenizeStateStart;
481 continue;
482 }
483 break;
484 case TokenizeStateSawPercent:
485 switch (c) {
486 case '=':
487 t.cur_tok->id = TokenIdModEq;
488 end_token(&t);
489 t.state = TokenizeStateStart;
490 break;
491 default:
492 t.pos -= 1;
493 end_token(&t);
494 t.state = TokenizeStateStart;
495 continue;
496 }
497 break;
498 case TokenizeStateSawPlus:
499 switch (c) {
500 case '=':
501 t.cur_tok->id = TokenIdPlusEq;
502 end_token(&t);
503 t.state = TokenizeStateStart;
504 break;
505 default:
506 t.pos -= 1;
507 end_token(&t);
508 t.state = TokenizeStateStart;
509 continue;
510 }
511 break;
512 case TokenizeStateSawAmpersand:
442 switch (c) {513 switch (c) {
443 case '&':514 case '&':
444 t.cur_tok->id = TokenIdBoolAnd;515 t.cur_tok->id = TokenIdBoolAnd;
516 t.state = TokenizeStateSawAmpersandAmpersand;
517 break;
518 case '=':
519 t.cur_tok->id = TokenIdBitAndEq;
445 end_token(&t);520 end_token(&t);
446 t.state = TokenizeStateStart;521 t.state = TokenizeStateStart;
447 break;522 break;
...@@ -452,10 +527,56 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -452,10 +527,56 @@ void tokenize(Buf *buf, Tokenization *out) {
452 continue;527 continue;
453 }528 }
454 break;529 break;
455 case TokenizeStatePipe:530 case TokenizeStateSawAmpersandAmpersand:
531 switch (c) {
532 case '=':
533 t.cur_tok->id = TokenIdBoolAndEq;
534 end_token(&t);
535 t.state = TokenizeStateStart;
536 break;
537 default:
538 t.pos -= 1;
539 end_token(&t);
540 t.state = TokenizeStateStart;
541 continue;
542 }
543 break;
544 case TokenizeStateSawCaret:
545 switch (c) {
546 case '=':
547 t.cur_tok->id = TokenIdBitXorEq;
548 end_token(&t);
549 t.state = TokenizeStateStart;
550 break;
551 default:
552 t.pos -= 1;
553 end_token(&t);
554 t.state = TokenizeStateStart;
555 continue;
556 }
557 break;
558 case TokenizeStateSawPipe:
456 switch (c) {559 switch (c) {
457 case '|':560 case '|':
458 t.cur_tok->id = TokenIdBoolOr;561 t.cur_tok->id = TokenIdBoolOr;
562 t.state = TokenizeStateSawPipePipe;
563 break;
564 case '=':
565 t.cur_tok->id = TokenIdBitOrEq;
566 end_token(&t);
567 t.state = TokenizeStateStart;
568 break;
569 default:
570 t.pos -= 1;
571 end_token(&t);
572 t.state = TokenizeStateStart;
573 continue;
574 }
575 break;
576 case TokenizeStateSawPipePipe:
577 switch (c) {
578 case '=':
579 t.cur_tok->id = TokenIdBoolOrEq;
459 end_token(&t);580 end_token(&t);
460 t.state = TokenizeStateStart;581 t.state = TokenizeStateStart;
461 break;582 break;
...@@ -477,6 +598,11 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -477,6 +598,11 @@ void tokenize(Buf *buf, Tokenization *out) {
477 t.state = TokenizeStateMultiLineComment;598 t.state = TokenizeStateMultiLineComment;
478 t.multi_line_comment_count = 1;599 t.multi_line_comment_count = 1;
479 break;600 break;
601 case '=':
602 t.cur_tok->id = TokenIdDivEq;
603 end_token(&t);
604 t.state = TokenizeStateStart;
605 break;
480 default:606 default:
481 t.pos -= 1;607 t.pos -= 1;
482 end_token(&t);608 end_token(&t);
...@@ -592,6 +718,11 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -592,6 +718,11 @@ void tokenize(Buf *buf, Tokenization *out) {
592 end_token(&t);718 end_token(&t);
593 t.state = TokenizeStateStart;719 t.state = TokenizeStateStart;
594 break;720 break;
721 case '=':
722 t.cur_tok->id = TokenIdMinusEq;
723 end_token(&t);
724 t.state = TokenizeStateStart;
725 break;
595 default:726 default:
596 t.pos -= 1;727 t.pos -= 1;
597 end_token(&t);728 end_token(&t);
...@@ -619,18 +750,26 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -619,18 +750,26 @@ void tokenize(Buf *buf, Tokenization *out) {
619 case TokenizeStateSymbol:750 case TokenizeStateSymbol:
620 case TokenizeStateSymbolFirst:751 case TokenizeStateSymbolFirst:
621 case TokenizeStateNumber:752 case TokenizeStateNumber:
753 case TokenizeStateSawStar:
754 case TokenizeStateSawSlash:
755 case TokenizeStateSawPercent:
756 case TokenizeStateSawPlus:
622 case TokenizeStateSawDash:757 case TokenizeStateSawDash:
623 case TokenizeStatePipe:758 case TokenizeStateSawAmpersand:
624 case TokenizeStateAmpersand:759 case TokenizeStateSawAmpersandAmpersand:
625 case TokenizeStateEq:760 case TokenizeStateSawCaret:
626 case TokenizeStateBang:761 case TokenizeStateSawPipe:
627 case TokenizeStateLessThan:762 case TokenizeStateSawPipePipe:
628 case TokenizeStateGreaterThan:763 case TokenizeStateSawEq:
629 case TokenizeStateDot:764 case TokenizeStateSawBang:
765 case TokenizeStateSawLessThan:
766 case TokenizeStateSawLessThanLessThan:
767 case TokenizeStateSawGreaterThan:
768 case TokenizeStateSawGreaterThanGreaterThan:
769 case TokenizeStateSawDot:
630 end_token(&t);770 end_token(&t);
631 break;771 break;
632 case TokenizeStateSawSlash:772 case TokenizeStateSawDotDot:
633 case TokenizeStateDotDot:
634 tokenize_error(&t, "unexpected EOF");773 tokenize_error(&t, "unexpected EOF");
635 break;774 break;
636 case TokenizeStateLineComment:775 case TokenizeStateLineComment:
...@@ -695,6 +834,18 @@ static const char * token_name(Token *token) {...@@ -695,6 +834,18 @@ static const char * token_name(Token *token) {
695 case TokenIdBoolOr: return "BoolOr";834 case TokenIdBoolOr: return "BoolOr";
696 case TokenIdBoolAnd: return "BoolAnd";835 case TokenIdBoolAnd: return "BoolAnd";
697 case TokenIdEq: return "Eq";836 case TokenIdEq: return "Eq";
837 case TokenIdTimesEq: return "TimesEq";
838 case TokenIdDivEq: return "DivEq";
839 case TokenIdModEq: return "ModEq";
840 case TokenIdPlusEq: return "PlusEq";
841 case TokenIdMinusEq: return "MinusEq";
842 case TokenIdBitShiftLeftEq: return "BitShiftLeftEq";
843 case TokenIdBitShiftRightEq: return "BitShiftRightEq";
844 case TokenIdBitAndEq: return "BitAndEq";
845 case TokenIdBitXorEq: return "BitXorEq";
846 case TokenIdBitOrEq: return "BitOrEq";
847 case TokenIdBoolAndEq: return "BoolAndEq";
848 case TokenIdBoolOrEq: return "BoolOrEq";
698 case TokenIdBang: return "Bang";849 case TokenIdBang: return "Bang";
699 case TokenIdTilde: return "Tilde";850 case TokenIdTilde: return "Tilde";
700 case TokenIdCmpEq: return "CmpEq";851 case TokenIdCmpEq: return "CmpEq";
src/tokenizer.hpp+12
...@@ -55,6 +55,18 @@ enum TokenId {...@@ -55,6 +55,18 @@ enum TokenId {
55 TokenIdBinAnd,55 TokenIdBinAnd,
56 TokenIdBinXor,56 TokenIdBinXor,
57 TokenIdEq,57 TokenIdEq,
58 TokenIdTimesEq,
59 TokenIdDivEq,
60 TokenIdModEq,
61 TokenIdPlusEq,
62 TokenIdMinusEq,
63 TokenIdBitShiftLeftEq,
64 TokenIdBitShiftRightEq,
65 TokenIdBitAndEq,
66 TokenIdBitXorEq,
67 TokenIdBitOrEq,
68 TokenIdBoolAndEq,
69 TokenIdBoolOrEq,
58 TokenIdCmpEq,70 TokenIdCmpEq,
59 TokenIdBang,71 TokenIdBang,
60 TokenIdTilde,72 TokenIdTilde,