authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-15 04:05:43-07:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2015-12-15 04:05:53-07:00
logf2a9b40231f982b10c4d098c60b47da890396c1f
tree63a8b8af9120203ff6f7dfaa05681346276e3c94
parentcf88fcb2ad31b64296d906ef525e76bbe6c3f65e

more number literal syntax is supported. floats still need work

7 files changed, 139 insertions(+), 221 deletions(-)

src/analyze.cpp+1-13
...@@ -640,13 +640,7 @@ static bool num_lit_fits_in_other_type(CodeGen *g, TypeTableEntry *literal_type,...@@ -640,13 +640,7 @@ static bool num_lit_fits_in_other_type(CodeGen *g, TypeTableEntry *literal_type,
640 case TypeTableEntryIdStruct:640 case TypeTableEntryIdStruct:
641 return false;641 return false;
642 case TypeTableEntryIdInt:642 case TypeTableEntryIdInt:
643 if (is_num_lit_signed(num_lit)) {643 if (is_num_lit_unsigned(num_lit)) {
644 if (!other_type->data.integral.is_signed) {
645 return false;
646 }
647
648 return lit_size_in_bits <= other_type->size_in_bits;
649 } else if (is_num_lit_unsigned(num_lit)) {
650644
651 return lit_size_in_bits <= other_type->size_in_bits;645 return lit_size_in_bits <= other_type->size_in_bits;
652 } else {646 } else {
...@@ -883,12 +877,6 @@ static TypeTableEntry * resolve_number_literals(CodeGen *g, AstNode *node1, AstN...@@ -883,12 +877,6 @@ static TypeTableEntry * resolve_number_literals(CodeGen *g, AstNode *node1, AstN
883 codegen_num_lit_1->resolved_type = g->builtin_types.entry_f64;877 codegen_num_lit_1->resolved_type = g->builtin_types.entry_f64;
884 codegen_num_lit_2->resolved_type = g->builtin_types.entry_f64;878 codegen_num_lit_2->resolved_type = g->builtin_types.entry_f64;
885 return g->builtin_types.entry_f64;879 return g->builtin_types.entry_f64;
886 } else if (is_num_lit_signed(type1->data.num_lit.kind) &&
887 is_num_lit_signed(type2->data.num_lit.kind))
888 {
889 codegen_num_lit_1->resolved_type = g->builtin_types.entry_i64;
890 codegen_num_lit_2->resolved_type = g->builtin_types.entry_i64;
891 return g->builtin_types.entry_i64;
892 } else if (is_num_lit_unsigned(type1->data.num_lit.kind) &&880 } else if (is_num_lit_unsigned(type1->data.num_lit.kind) &&
893 is_num_lit_unsigned(type2->data.num_lit.kind))881 is_num_lit_unsigned(type2->data.num_lit.kind))
894 {882 {
src/codegen.cpp+1-5
...@@ -1254,13 +1254,9 @@ static const NumLit num_lit_kinds[] = {...@@ -1254,13 +1254,9 @@ static const NumLit num_lit_kinds[] = {
1254 NumLitF32,1254 NumLitF32,
1255 NumLitF64,1255 NumLitF64,
1256 NumLitF128,1256 NumLitF128,
1257 NumLitI8,
1258 NumLitU8,1257 NumLitU8,
1259 NumLitI16,
1260 NumLitU16,1258 NumLitU16,
1261 NumLitI32,
1262 NumLitU32,1259 NumLitU32,
1263 NumLitI64,
1264 NumLitU64,1260 NumLitU64,
1265};1261};
12661262
...@@ -1388,7 +1384,7 @@ static void define_builtin_types(CodeGen *g) {...@@ -1388,7 +1384,7 @@ static void define_builtin_types(CodeGen *g) {
1388 }1384 }
1389 {1385 {
1390 TypeTableEntry *entry = new_type_table_entry(TypeTableEntryIdFloat);1386 TypeTableEntry *entry = new_type_table_entry(TypeTableEntryIdFloat);
1391 entry->type_ref = LLVMFloatType();1387 entry->type_ref = LLVMDoubleType();
1392 buf_init_from_str(&entry->name, "f64");1388 buf_init_from_str(&entry->name, "f64");
1393 entry->size_in_bits = 64;1389 entry->size_in_bits = 64;
1394 entry->align_in_bits = 64;1390 entry->align_in_bits = 64;
src/parser.cpp+118-196
...@@ -11,6 +11,7 @@...@@ -11,6 +11,7 @@
1111
12#include <stdarg.h>12#include <stdarg.h>
13#include <stdio.h>13#include <stdio.h>
14#include <limits.h>
1415
1516
16static const char *bin_op_str(BinOpType bin_op) {17static const char *bin_op_str(BinOpType bin_op) {
...@@ -278,9 +279,7 @@ void ast_print(AstNode *node, int indent) {...@@ -278,9 +279,7 @@ void ast_print(AstNode *node, int indent) {
278 NumLit num_lit = node->data.number_literal.kind;279 NumLit num_lit = node->data.number_literal.kind;
279 const char *name = node_type_str(node->type);280 const char *name = node_type_str(node->type);
280 const char *kind_str = num_lit_str(num_lit);281 const char *kind_str = num_lit_str(num_lit);
281 if (is_num_lit_signed(num_lit)) {282 if (is_num_lit_unsigned(num_lit)) {
282 fprintf(stderr, "%s %s %" PRId64 "\n", name, kind_str, node->data.number_literal.data.x_int);
283 } else if (is_num_lit_unsigned(num_lit)) {
284 fprintf(stderr, "%s %s %" PRIu64 "\n", name, kind_str, node->data.number_literal.data.x_uint);283 fprintf(stderr, "%s %s %" PRIu64 "\n", name, kind_str, node->data.number_literal.data.x_uint);
285 } else {284 } else {
286 fprintf(stderr, "%s %s %f\n", name, kind_str, node->data.number_literal.data.x_float);285 fprintf(stderr, "%s %s %f\n", name, kind_str, node->data.number_literal.data.x_float);
...@@ -585,187 +584,152 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool...@@ -585,187 +584,152 @@ static void parse_string_literal(ParseContext *pc, Token *token, Buf *buf, bool
585 if (offset_map) offset_map->append(pos);584 if (offset_map) offset_map->append(pos);
586}585}
587586
588enum ParseNumLitState {587static unsigned long long parse_int_digits(ParseContext *pc, int digits_start, int digits_end, int radix,
589 ParseNumLitStateStart,588 unsigned long long initial_value, bool *overflow) {
590 ParseNumLitStateBase,589 unsigned long long x = initial_value;
591 ParseNumLitStateDigits,
592 ParseNumLitStateExpectFirstDigit,
593 ParseNumLitStateDecimal,
594 ParseNumLitStateESign,
595 ParseNumLitStateEDigit,
596};
597590
598static void parse_number_literal(ParseContext *pc, Token *token, AstNodeNumberLiteral *num_lit) {591 for (int i = digits_start; i < digits_end; i++) {
599 ParseNumLitState state = ParseNumLitStateStart;
600 unsigned long long base = 10;
601 bool negative = false;
602 int digits_start;
603 int digits_end;
604 int decimal_start = -1;
605 int decimal_end;
606 bool e_present = false;
607 bool e_positive;
608 int e_digit_start;
609 int e_digit_end;
610
611 for (int i = token->start_pos; i < token->end_pos; i += 1) {
612 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);592 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);
613 switch (state) {593 unsigned long long digit = get_digit_value(c);
614 case ParseNumLitStateStart:
615 if (c == '-') {
616 negative = true;
617 } else if (c == '0') {
618 state = ParseNumLitStateBase;
619 } else if (c >= '1' && c <= '9') {
620 digits_start = i;
621 state = ParseNumLitStateDigits;
622 } else {
623 zig_unreachable();
624 }
625 break;
626 case ParseNumLitStateBase:
627 if (c == 'x') {
628 base = 16;
629 state = ParseNumLitStateExpectFirstDigit;
630 } else if (c == 'o') {
631 base = 8;
632 state = ParseNumLitStateExpectFirstDigit;
633 } else if (c == 'b') {
634 base = 2;
635 state = ParseNumLitStateExpectFirstDigit;
636 } else {
637 zig_unreachable();
638 }
639 break;
640594
641 case ParseNumLitStateExpectFirstDigit:595 // x *= radix;
642 state = ParseNumLitStateDigits;596 if (__builtin_umulll_overflow(x, radix, &x)) {
643 break;597 *overflow = true;
598 return 0;
599 }
644600
645 case ParseNumLitStateDigits:601 // x += digit
646 if (c == '.') {602 if (__builtin_uaddll_overflow(x, digit, &x)) {
647 assert(base == 10);603 *overflow = true;
648 digits_end = i;604 return 0;
649 decimal_start = i + 1;
650 state = ParseNumLitStateDecimal;
651 }
652 break;
653 case ParseNumLitStateDecimal:
654 if (c == 'E') {
655 e_present = false;
656 decimal_end = i;
657 state = ParseNumLitStateESign;
658 }
659 break;
660 case ParseNumLitStateESign:
661 if (c == '+') {
662 e_positive = true;
663 e_digit_start = i + 1;
664 state = ParseNumLitStateEDigit;
665 } else if (c == '-') {
666 e_positive = false;
667 e_digit_start = i + 1;
668 state = ParseNumLitStateEDigit;
669 } else {
670 zig_unreachable();
671 }
672 break;
673 case ParseNumLitStateEDigit:
674 assert(c >= '0' && c <= '9');
675 break;
676 }605 }
677 }606 }
607 return x;
608}
678609
679 switch (state) {610static void parse_number_literal(ParseContext *pc, Token *token, AstNodeNumberLiteral *num_lit) {
680 case ParseNumLitStateDigits:611 assert(token->id == TokenIdNumberLiteral);
681 digits_end = token->end_pos;612
682 break;613 int whole_number_start = token->start_pos;
683 case ParseNumLitStateDecimal:614 if (token->radix != 10) {
684 decimal_end = token->end_pos;615 // skip the "0x"
685 break;616 whole_number_start += 2;
686 case ParseNumLitStateEDigit:
687 e_digit_end = token->end_pos;
688 break;
689 case ParseNumLitStateBase:
690 num_lit->kind = NumLitU8;
691 num_lit->data.x_uint = 0;
692 return;
693 case ParseNumLitStateESign:
694 case ParseNumLitStateExpectFirstDigit:
695 case ParseNumLitStateStart:
696 zig_unreachable();
697 }617 }
698618
699 if (decimal_start >= 0) {619 int whole_number_end = token->decimal_point_pos;
700 // float620 if (whole_number_end <= whole_number_start) {
701 double x;621 // TODO: error for empty whole number part
702622 return;
703 (void)x;623 }
704 (void)decimal_end;624
705 (void)e_present;625 if (token->decimal_point_pos == token->end_pos) {
706 (void)e_positive;
707 (void)e_digit_start;
708 (void)e_digit_end;
709 zig_panic("TODO parse float");
710 } else {
711 // integer626 // integer
712 unsigned long long x = 0;627 unsigned long long whole_number = parse_int_digits(pc, whole_number_start, whole_number_end,
628 token->radix, 0, &num_lit->overflow);
629 if (num_lit->overflow) return;
713630
714 unsigned long long mult = 1;631 num_lit->data.x_uint = whole_number;
715 for (int i = digits_end - 1; ; i -= 1) {
716 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + i);
717 unsigned long long digit = (c - '0');
718632
719 // digit *= mult633 if (whole_number <= UINT8_MAX) {
720 if (__builtin_umulll_overflow(digit, mult, &digit)) {634 num_lit->kind = NumLitU8;
721 num_lit->overflow = true;635 } else if (whole_number <= UINT16_MAX) {
636 num_lit->kind = NumLitU16;
637 } else if (whole_number <= UINT32_MAX) {
638 num_lit->kind = NumLitU32;
639 } else {
640 num_lit->kind = NumLitU64;
641 }
642 } else {
643 // float
644 // TODO: trim leading and trailing zeros in the significand digit sequence
645 unsigned long long significand_as_int = parse_int_digits(pc, whole_number_start, whole_number_end,
646 token->radix, 0, &num_lit->overflow);
647 if (num_lit->overflow) return;
648
649 int exponent = 0;
650 if (token->decimal_point_pos < token->exponent_marker_pos) {
651 // fraction
652 int fraction_start = token->decimal_point_pos + 1;
653 int fraction_end = token->exponent_marker_pos;
654 if (fraction_end <= fraction_start) {
655 // TODO: error for empty fraction part
722 return;656 return;
723 }657 }
724658
725 // x += digit659 // TODO: check for where the fraction got too precise instead of just saying overflow
726 if (__builtin_uaddll_overflow(x, digit, &x)) {660 significand_as_int = parse_int_digits(pc, fraction_start, fraction_end,
727 num_lit->overflow = true;661 token->radix, significand_as_int, &num_lit->overflow);
662 if (num_lit->overflow) return;
663
664 // adjust the exponent to compensate for us effectively moving
665 // the decimal point all the way to the right
666 exponent = -(fraction_end - fraction_start);
667 }
668
669 if (token->exponent_marker_pos < token->end_pos) {
670 // exponent
671 int exponent_start = token->exponent_marker_pos + 1;
672 int exponent_end = token->end_pos;
673 if (exponent_end <= exponent_start) {
674 // TODO: error for empty exponent part
728 return;675 return;
729 }676 }
730677
731 if (i == digits_start)678 bool is_exponent_negative = false;
732 break;679 uint8_t c = *((uint8_t*)buf_ptr(pc->buf) + exponent_start);
680 if (c == '+') {
681 exponent_start += 1;
682 } else if (c == '-') {
683 exponent_start += 1;
684 is_exponent_negative = true;
685 }
733686
734 // mult *= base687 if (exponent_end <= exponent_start) {
735 if (__builtin_umulll_overflow(mult, base, &mult)) {688 // TODO: error for empty exponent part
689 return;
690 }
691
692 unsigned long long specified_exponent = parse_int_digits(pc, exponent_start, exponent_end,
693 10, 0, &num_lit->overflow);
694 // TODO: this check is a little silly
695 if (specified_exponent >= LONG_LONG_MAX) {
736 num_lit->overflow = true;696 num_lit->overflow = true;
737 return;697 return;
738 }698 }
699 if (is_exponent_negative) {
700 exponent -= specified_exponent;
701 } else {
702 exponent += specified_exponent;
703 }
739 }704 }
740705
741 if (negative) {706 uint64_t significand_bits;
742 if (x <= 128ull) {707 uint64_t exponent_bits;
743 num_lit->kind = NumLitI8;708 if (significand_as_int != 0) {
744 } else if (x <= 32768ull) {709 // normalize the significand
745 num_lit->kind = NumLitI16;710 int significand_magnitude = __builtin_clzll(1) - __builtin_clzll(significand_as_int);
746 } else if (x <= 2147483648ull) {711 exponent += significand_magnitude;
747 num_lit->kind = NumLitI32;712 if (!(-1023 <= exponent && exponent < 1023)) {
748 } else if (x <= 9223372036854775808ull) {
749 num_lit->kind = NumLitI64;
750 } else {
751 num_lit->overflow = true;713 num_lit->overflow = true;
752 return;714 return;
753 }715 }
754716
755 num_lit->data.x_int = -((int64_t)x);717 // this should chop off exactly one 1 bit from the top.
718 significand_bits = ((uint64_t)significand_as_int << (52 - significand_magnitude)) & 0xfffffffffffffULL;
719 exponent_bits = exponent + 1023;
756 } else {720 } else {
757 num_lit->data.x_uint = x;721 // 0 is all 0's
758722 significand_bits = 0;
759 if (x <= UINT8_MAX) {723 exponent_bits = 0;
760 num_lit->kind = NumLitU8;
761 } else if (x <= UINT16_MAX) {
762 num_lit->kind = NumLitU16;
763 } else if (x <= UINT32_MAX) {
764 num_lit->kind = NumLitU32;
765 } else {
766 num_lit->kind = NumLitU64;
767 }
768 }724 }
725
726 uint64_t double_bits = (exponent_bits << 52) | significand_bits;
727 // TODO: check and swap endian
728 double x = *(double *)&double_bits;
729
730 num_lit->data.x_float = x;
731 // TODO: see if we can store it in f32
732 num_lit->kind = NumLitF64;
769 }733 }
770}734}
771735
...@@ -2366,14 +2330,6 @@ const char *num_lit_str(NumLit num_lit) {...@@ -2366,14 +2330,6 @@ const char *num_lit_str(NumLit num_lit) {
2366 return "f64";2330 return "f64";
2367 case NumLitF128:2331 case NumLitF128:
2368 return "f128";2332 return "f128";
2369 case NumLitI8:
2370 return "i8";
2371 case NumLitI16:
2372 return "i16";
2373 case NumLitI32:
2374 return "i32";
2375 case NumLitI64:
2376 return "i64";
2377 case NumLitU8:2333 case NumLitU8:
2378 return "u8";2334 return "u8";
2379 case NumLitU16:2335 case NumLitU16:
...@@ -2388,37 +2344,11 @@ const char *num_lit_str(NumLit num_lit) {...@@ -2388,37 +2344,11 @@ const char *num_lit_str(NumLit num_lit) {
2388 zig_unreachable();2344 zig_unreachable();
2389}2345}
23902346
2391bool is_num_lit_signed(NumLit num_lit) {
2392 switch (num_lit) {
2393 case NumLitI8:
2394 case NumLitI16:
2395 case NumLitI32:
2396 case NumLitI64:
2397 return true;
2398
2399 case NumLitF32:
2400 case NumLitF64:
2401 case NumLitF128:
2402 case NumLitU8:
2403 case NumLitU16:
2404 case NumLitU32:
2405 case NumLitU64:
2406 return false;
2407 case NumLitCount:
2408 zig_unreachable();
2409 }
2410 zig_unreachable();
2411}
2412
2413bool is_num_lit_unsigned(NumLit num_lit) {2347bool is_num_lit_unsigned(NumLit num_lit) {
2414 switch (num_lit) {2348 switch (num_lit) {
2415 case NumLitF32:2349 case NumLitF32:
2416 case NumLitF64:2350 case NumLitF64:
2417 case NumLitF128:2351 case NumLitF128:
2418 case NumLitI8:
2419 case NumLitI16:
2420 case NumLitI32:
2421 case NumLitI64:
2422 return false;2352 return false;
2423 case NumLitU8:2353 case NumLitU8:
2424 case NumLitU16:2354 case NumLitU16:
...@@ -2437,10 +2367,6 @@ bool is_num_lit_float(NumLit num_lit) {...@@ -2437,10 +2367,6 @@ bool is_num_lit_float(NumLit num_lit) {
2437 case NumLitF64:2367 case NumLitF64:
2438 case NumLitF128:2368 case NumLitF128:
2439 return true;2369 return true;
2440 case NumLitI8:
2441 case NumLitI16:
2442 case NumLitI32:
2443 case NumLitI64:
2444 case NumLitU8:2370 case NumLitU8:
2445 case NumLitU16:2371 case NumLitU16:
2446 case NumLitU32:2372 case NumLitU32:
...@@ -2454,17 +2380,13 @@ bool is_num_lit_float(NumLit num_lit) {...@@ -2454,17 +2380,13 @@ bool is_num_lit_float(NumLit num_lit) {
24542380
2455uint64_t num_lit_bit_count(NumLit num_lit) {2381uint64_t num_lit_bit_count(NumLit num_lit) {
2456 switch (num_lit) {2382 switch (num_lit) {
2457 case NumLitI8:
2458 case NumLitU8:2383 case NumLitU8:
2459 return 8;2384 return 8;
2460 case NumLitI16:
2461 case NumLitU16:2385 case NumLitU16:
2462 return 16;2386 return 16;
2463 case NumLitI32:
2464 case NumLitU32:2387 case NumLitU32:
2465 case NumLitF32:2388 case NumLitF32:
2466 return 32;2389 return 32;
2467 case NumLitI64:
2468 case NumLitU64:2390 case NumLitU64:
2469 case NumLitF64:2391 case NumLitF64:
2470 return 64;2392 return 64;
src/parser.hpp-6
...@@ -273,13 +273,9 @@ enum NumLit {...@@ -273,13 +273,9 @@ enum NumLit {
273 NumLitF32,273 NumLitF32,
274 NumLitF64,274 NumLitF64,
275 NumLitF128,275 NumLitF128,
276 NumLitI8,
277 NumLitU8,276 NumLitU8,
278 NumLitI16,
279 NumLitU16,277 NumLitU16,
280 NumLitI32,
281 NumLitU32,278 NumLitU32,
282 NumLitI64,
283 NumLitU64,279 NumLitU64,
284280
285 NumLitCount281 NumLitCount
...@@ -294,7 +290,6 @@ struct AstNodeNumberLiteral {...@@ -294,7 +290,6 @@ struct AstNodeNumberLiteral {
294290
295 union {291 union {
296 uint64_t x_uint;292 uint64_t x_uint;
297 int64_t x_int;
298 double x_float;293 double x_float;
299 } data;294 } data;
300};295};
...@@ -362,7 +357,6 @@ const char *node_type_str(NodeType node_type);...@@ -362,7 +357,6 @@ const char *node_type_str(NodeType node_type);
362void ast_print(AstNode *node, int indent);357void ast_print(AstNode *node, int indent);
363358
364const char *num_lit_str(NumLit num_lit);359const char *num_lit_str(NumLit num_lit);
365bool is_num_lit_signed(NumLit num_lit);
366bool is_num_lit_unsigned(NumLit num_lit);360bool is_num_lit_unsigned(NumLit num_lit);
367bool is_num_lit_float(NumLit num_lit);361bool is_num_lit_float(NumLit num_lit);
368uint64_t num_lit_bit_count(NumLit num_lit);362uint64_t num_lit_bit_count(NumLit num_lit);
src/tokenizer.cpp+5-1
...@@ -167,6 +167,9 @@ static void begin_token(Tokenize *t, TokenId id) {...@@ -167,6 +167,9 @@ static void begin_token(Tokenize *t, TokenId id) {
167 token->start_column = t->column;167 token->start_column = t->column;
168 token->id = id;168 token->id = id;
169 token->start_pos = t->pos;169 token->start_pos = t->pos;
170 token->radix = 0;
171 token->decimal_point_pos = 0;
172 token->exponent_marker_pos = 0;
170 t->cur_tok = token;173 t->cur_tok = token;
171}174}
172175
...@@ -242,7 +245,8 @@ static bool is_exponent_signifier(uint8_t c, int radix) {...@@ -242,7 +245,8 @@ static bool is_exponent_signifier(uint8_t c, int radix) {
242 return c == 'e' || c == 'E';245 return c == 'e' || c == 'E';
243 }246 }
244}247}
245static int get_digit_value(uint8_t c) {248
249int get_digit_value(uint8_t c) {
246 if ('0' <= c && c <= '9') {250 if ('0' <= c && c <= '9') {
247 return c - '0';251 return c - '0';
248 }252 }
src/tokenizer.hpp+1
...@@ -110,5 +110,6 @@ void tokenize(Buf *buf, Tokenization *out_tokenization);...@@ -110,5 +110,6 @@ void tokenize(Buf *buf, Tokenization *out_tokenization);
110void print_tokens(Buf *buf, ZigList<Token> *tokens);110void print_tokens(Buf *buf, ZigList<Token> *tokens);
111111
112bool is_printable(uint8_t c);112bool is_printable(uint8_t c);
113int get_digit_value(uint8_t c);
113114
114#endif115#endif
test/run_tests.cpp+13
...@@ -477,6 +477,19 @@ export fn main(argc : isize, argv : &&u8, env : &&u8) -> i32 {...@@ -477,6 +477,19 @@ export fn main(argc : isize, argv : &&u8, env : &&u8) -> i32 {
477}477}
478 )SOURCE", "OK\n");478 )SOURCE", "OK\n");
479479
480 add_simple_case("number literals", R"SOURCE(
481#link("c")
482extern {
483 fn printf(__format: &const u8, ...) -> i32;
484 fn exit(__status: i32) -> unreachable;
485}
486
487export fn _start() -> unreachable {
488 printf(c"0=%d\n", 0 as i32); // TODO: more tests
489 exit(0);
490}
491 )SOURCE", "0=0\n");
492
480 add_simple_case("structs", R"SOURCE(493 add_simple_case("structs", R"SOURCE(
481use "std.zig";494use "std.zig";
482495