authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:25:59-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:25:59-05:00
log07898cc0dfae880b35bededa726c7e16a3c45f3d
tree293135e757f9599c93440dcf594df6d820cd7f4c
parent798dbe487b678a85c7e10b78c2424c4b3508475b

tokenizing string literals

1 files changed, 50 insertions(+), 1 deletions(-)

src-self-hosted/main.zig+50-1
...@@ -64,12 +64,15 @@ const Token = struct {...@@ -64,12 +64,15 @@ const Token = struct {
64 return null;64 return null;
65 }65 }
6666
67 const StrLitKind = enum {Normal, C};
6768
68 const Id = union(enum) {69 const Id = union(enum) {
69 Invalid,70 Invalid,
70 Identifier,71 Identifier,
71 Keyword: Keyword,72 Keyword: Keyword,
73 StringLiteral: StrLitKind,
72 Eof,74 Eof,
75 Builtin,
73 };76 };
74};77};
7578
...@@ -91,6 +94,10 @@ const Tokenizer = struct {...@@ -91,6 +94,10 @@ const Tokenizer = struct {
91 const State = enum {94 const State = enum {
92 Start,95 Start,
93 Identifier,96 Identifier,
97 Builtin,
98 C,
99 StringLiteral,
100 StringLiteralBackslash,
94 };101 };
95102
96 pub fn next(self: &Tokenizer) -> Token {103 pub fn next(self: &Tokenizer) -> Token {
...@@ -107,10 +114,22 @@ const Tokenizer = struct {...@@ -107,10 +114,22 @@ const Tokenizer = struct {
107 ' ', '\n' => {114 ' ', '\n' => {
108 result.start = self.index + 1;115 result.start = self.index + 1;
109 },116 },
110 'a'...'z', 'A'...'Z', '_' => {117 'c' => {
118 state = State.C;
119 result.id = Token.Id { .Identifier = {} };
120 },
121 '"' => {
122 state = State.StringLiteral;
123 result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
124 },
125 'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
111 state = State.Identifier;126 state = State.Identifier;
112 result.id = Token.Id { .Identifier = {} };127 result.id = Token.Id { .Identifier = {} };
113 },128 },
129 '@' => {
130 state = State.Builtin;
131 result.id = Token.Id { .Builtin = {} };
132 },
114 else => {133 else => {
115 result.id = Token.Id { .Invalid = {} };134 result.id = Token.Id { .Invalid = {} };
116 self.index += 1;135 self.index += 1;
...@@ -126,6 +145,36 @@ const Tokenizer = struct {...@@ -126,6 +145,36 @@ const Tokenizer = struct {
126 break;145 break;
127 },146 },
128 },147 },
148 State.Builtin => switch (c) {
149 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
150 else => break,
151 },
152 State.C => switch (c) {
153 '\\' => @panic("TODO"),
154 '"' => {
155 state = State.StringLiteral;
156 result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
157 },
158 'a'...'z', 'A'...'Z', '_', '0'...'9' => {
159 state = State.Identifier;
160 },
161 else => break,
162 },
163 State.StringLiteral => switch (c) {
164 '\\' => {
165 state = State.StringLiteralBackslash;
166 },
167 '"' => {
168 self.index += 1;
169 break;
170 },
171 '\n' => break, // Look for this error later.
172 else => {},
173 },
174 State.StringLiteralBackslash => switch (c) {
175 '\n' => break, // Look for this error later.
176 else => {},
177 },
129 }178 }
130 }179 }
131 result.end = self.index;180 result.end = self.index;