authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-24 01:15:50-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-24 01:16:07-04:00
log54e887ed9e774c6fd68f51d97e2c5c760e48be30
treeafcbee7cecdde666b42a928f82748367d92d57ad
parentb132a17a749c27040c7fed43b268c2f0d9a401ce

std.zig.tokenizer: fix tokenization of hex floats

2 files changed, 77 insertions(+), 13 deletions(-)

std/zig/parser_test.zig+8-7
...@@ -1,3 +1,11 @@...@@ -1,3 +1,11 @@
1test "zig fmt: float literal with exponent" {
2 try testCanonical(
3 \\pub const f64_true_min = 4.94065645841246544177e-324;
4 \\const threshold = 0x1.a827999fcef32p+1022;
5 \\
6 );
7}
8
1test "zig fmt: if-else end of comptime" {9test "zig fmt: if-else end of comptime" {
2 try testCanonical(10 try testCanonical(
3 \\comptime {11 \\comptime {
...@@ -238,13 +246,6 @@ test "zig fmt: switch with empty body" {...@@ -238,13 +246,6 @@ test "zig fmt: switch with empty body" {
238 );246 );
239}247}
240248
241test "zig fmt: float literal with exponent" {
242 try testCanonical(
243 \\pub const f64_true_min = 4.94065645841246544177e-324;
244 \\
245 );
246}
247
248test "zig fmt: line comments in struct initializer" {249test "zig fmt: line comments in struct initializer" {
249 try testCanonical(250 try testCanonical(
250 \\fn foo() void {251 \\fn foo() void {
std/zig/tokenizer.zig+69-6
...@@ -6,12 +6,12 @@ pub const Token = struct {...@@ -6,12 +6,12 @@ pub const Token = struct {
6 start: usize,6 start: usize,
7 end: usize,7 end: usize,
88
9 const Keyword = struct {9 pub const Keyword = struct {
10 bytes: []const u8,10 bytes: []const u8,
11 id: Id,11 id: Id,
12 };12 };
1313
14 const keywords = []Keyword {14 pub const keywords = []Keyword {
15 Keyword{.bytes="align", .id = Id.Keyword_align},15 Keyword{.bytes="align", .id = Id.Keyword_align},
16 Keyword{.bytes="and", .id = Id.Keyword_and},16 Keyword{.bytes="and", .id = Id.Keyword_and},
17 Keyword{.bytes="asm", .id = Id.Keyword_asm},17 Keyword{.bytes="asm", .id = Id.Keyword_asm},
...@@ -62,6 +62,7 @@ pub const Token = struct {...@@ -62,6 +62,7 @@ pub const Token = struct {
62 Keyword{.bytes="while", .id = Id.Keyword_while},62 Keyword{.bytes="while", .id = Id.Keyword_while},
63 };63 };
6464
65 // TODO perfect hash at comptime
65 fn getKeyword(bytes: []const u8) ?Id {66 fn getKeyword(bytes: []const u8) ?Id {
66 for (keywords) |kw| {67 for (keywords) |kw| {
67 if (mem.eql(u8, kw.bytes, bytes)) {68 if (mem.eql(u8, kw.bytes, bytes)) {
...@@ -236,10 +237,15 @@ pub const Tokenizer = struct {...@@ -236,10 +237,15 @@ pub const Tokenizer = struct {
236 Zero,237 Zero,
237 IntegerLiteral,238 IntegerLiteral,
238 IntegerLiteralWithRadix,239 IntegerLiteralWithRadix,
240 IntegerLiteralWithRadixHex,
239 NumberDot,241 NumberDot,
242 NumberDotHex,
240 FloatFraction,243 FloatFraction,
244 FloatFractionHex,
241 FloatExponentUnsigned,245 FloatExponentUnsigned,
246 FloatExponentUnsignedHex,
242 FloatExponentNumber,247 FloatExponentNumber,
248 FloatExponentNumberHex,
243 Ampersand,249 Ampersand,
244 Caret,250 Caret,
245 Percent,251 Percent,
...@@ -839,9 +845,12 @@ pub const Tokenizer = struct {...@@ -839,9 +845,12 @@ pub const Tokenizer = struct {
839 else => self.checkLiteralCharacter(),845 else => self.checkLiteralCharacter(),
840 },846 },
841 State.Zero => switch (c) {847 State.Zero => switch (c) {
842 'b', 'o', 'x' => {848 'b', 'o' => {
843 state = State.IntegerLiteralWithRadix;849 state = State.IntegerLiteralWithRadix;
844 },850 },
851 'x' => {
852 state = State.IntegerLiteralWithRadixHex;
853 },
845 else => {854 else => {
846 // reinterpret as a normal number855 // reinterpret as a normal number
847 self.index -= 1;856 self.index -= 1;
...@@ -862,8 +871,15 @@ pub const Tokenizer = struct {...@@ -862,8 +871,15 @@ pub const Tokenizer = struct {
862 '.' => {871 '.' => {
863 state = State.NumberDot;872 state = State.NumberDot;
864 },873 },
874 '0'...'9' => {},
875 else => break,
876 },
877 State.IntegerLiteralWithRadixHex => switch (c) {
878 '.' => {
879 state = State.NumberDotHex;
880 },
865 'p', 'P' => {881 'p', 'P' => {
866 state = State.FloatExponentUnsigned;882 state = State.FloatExponentUnsignedHex;
867 },883 },
868 '0'...'9', 'a'...'f', 'A'...'F' => {},884 '0'...'9', 'a'...'f', 'A'...'F' => {},
869 else => break,885 else => break,
...@@ -880,13 +896,32 @@ pub const Tokenizer = struct {...@@ -880,13 +896,32 @@ pub const Tokenizer = struct {
880 state = State.FloatFraction;896 state = State.FloatFraction;
881 },897 },
882 },898 },
899 State.NumberDotHex => switch (c) {
900 '.' => {
901 self.index -= 1;
902 state = State.Start;
903 break;
904 },
905 else => {
906 self.index -= 1;
907 result.id = Token.Id.FloatLiteral;
908 state = State.FloatFractionHex;
909 },
910 },
883 State.FloatFraction => switch (c) {911 State.FloatFraction => switch (c) {
884 'p', 'P', 'e', 'E' => {912 'e', 'E' => {
885 state = State.FloatExponentUnsigned;913 state = State.FloatExponentUnsigned;
886 },914 },
887 '0'...'9' => {},915 '0'...'9' => {},
888 else => break,916 else => break,
889 },917 },
918 State.FloatFractionHex => switch (c) {
919 'p', 'P' => {
920 state = State.FloatExponentUnsignedHex;
921 },
922 '0'...'9', 'a'...'f', 'A'...'F' => {},
923 else => break,
924 },
890 State.FloatExponentUnsigned => switch (c) {925 State.FloatExponentUnsigned => switch (c) {
891 '+', '-' => {926 '+', '-' => {
892 state = State.FloatExponentNumber;927 state = State.FloatExponentNumber;
...@@ -897,7 +932,21 @@ pub const Tokenizer = struct {...@@ -897,7 +932,21 @@ pub const Tokenizer = struct {
897 state = State.FloatExponentNumber;932 state = State.FloatExponentNumber;
898 }933 }
899 },934 },
935 State.FloatExponentUnsignedHex => switch (c) {
936 '+', '-' => {
937 state = State.FloatExponentNumberHex;
938 },
939 else => {
940 // reinterpret as a normal exponent number
941 self.index -= 1;
942 state = State.FloatExponentNumberHex;
943 }
944 },
900 State.FloatExponentNumber => switch (c) {945 State.FloatExponentNumber => switch (c) {
946 '0'...'9' => {},
947 else => break,
948 },
949 State.FloatExponentNumberHex => switch (c) {
901 '0'...'9', 'a'...'f', 'A'...'F' => {},950 '0'...'9', 'a'...'f', 'A'...'F' => {},
902 else => break,951 else => break,
903 },952 },
...@@ -908,8 +957,11 @@ pub const Tokenizer = struct {...@@ -908,8 +957,11 @@ pub const Tokenizer = struct {
908 State.C,957 State.C,
909 State.IntegerLiteral,958 State.IntegerLiteral,
910 State.IntegerLiteralWithRadix,959 State.IntegerLiteralWithRadix,
960 State.IntegerLiteralWithRadixHex,
911 State.FloatFraction,961 State.FloatFraction,
962 State.FloatFractionHex,
912 State.FloatExponentNumber,963 State.FloatExponentNumber,
964 State.FloatExponentNumberHex,
913 State.StringLiteral, // find this error later965 State.StringLiteral, // find this error later
914 State.MultilineStringLiteralLine,966 State.MultilineStringLiteralLine,
915 State.Builtin => {},967 State.Builtin => {},
...@@ -928,7 +980,9 @@ pub const Tokenizer = struct {...@@ -928,7 +980,9 @@ pub const Tokenizer = struct {
928 },980 },
929981
930 State.NumberDot,982 State.NumberDot,
983 State.NumberDotHex,
931 State.FloatExponentUnsigned,984 State.FloatExponentUnsigned,
985 State.FloatExponentUnsignedHex,
932 State.SawAtSign,986 State.SawAtSign,
933 State.Backslash,987 State.Backslash,
934 State.MultilineStringLiteralLineBackslash,988 State.MultilineStringLiteralLineBackslash,
...@@ -1073,7 +1127,7 @@ test "tokenizer" {...@@ -1073,7 +1127,7 @@ test "tokenizer" {
1073 });1127 });
1074}1128}
10751129
1076test "tokenizer - float literal" {1130test "tokenizer - float literal e exponent" {
1077 testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id {1131 testTokenize("a = 4.94065645841246544177e-324;\n", []Token.Id {
1078 Token.Id.Identifier,1132 Token.Id.Identifier,
1079 Token.Id.Equal,1133 Token.Id.Equal,
...@@ -1082,6 +1136,15 @@ test "tokenizer - float literal" {...@@ -1082,6 +1136,15 @@ test "tokenizer - float literal" {
1082 });1136 });
1083}1137}
10841138
1139test "tokenizer - float literal p exponent" {
1140 testTokenize("a = 0x1.a827999fcef32p+1022;\n", []Token.Id {
1141 Token.Id.Identifier,
1142 Token.Id.Equal,
1143 Token.Id.FloatLiteral,
1144 Token.Id.Semicolon,
1145 });
1146}
1147
1085test "tokenizer - chars" {1148test "tokenizer - chars" {
1086 testTokenize("'c'", []Token.Id {Token.Id.CharLiteral});1149 testTokenize("'c'", []Token.Id {Token.Id.CharLiteral});
1087}1150}