authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-04 10:27:38+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-04 10:27:38+02:00
log020724cfa0677bf42f48957d0ca7a474f6fe31e0
tree6c9763a6bac0a5daed355ae758de6109addd729d
parent09cf82361999c1a22819467e02fc68fd22ca188e

std.zig.tokenizer Tokens now don't contain a line and column field.

* Instead, this information is optained by asking the tokenizer. * getTokenLocation takes a start_index, so relative loc can be optained

2 files changed, 25 insertions(+), 42 deletions(-)

std/zig/parser.zig+10-13
...@@ -1571,12 +1571,12 @@ pub const Parser = struct {...@@ -1571,12 +1571,12 @@ pub const Parser = struct {
1571 }1571 }
15721572
1573 fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) {1573 fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) {
1574 const loc = self.tokenizer.getTokenLocation(token);1574 const loc = self.tokenizer.getTokenLocation(0, token);
1575 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args);1575 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
1576 warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);1576 warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
1577 {1577 {
1578 var i: usize = 0;1578 var i: usize = 0;
1579 while (i < token.column) : (i += 1) {1579 while (i < loc.column) : (i += 1) {
1580 warn(" ");1580 warn(" ");
1581 }1581 }
1582 }1582 }
...@@ -1679,9 +1679,8 @@ pub const Parser = struct {...@@ -1679,9 +1679,8 @@ pub const Parser = struct {
1679 try stack.append(RenderState {1679 try stack.append(RenderState {
1680 .Text = blk: {1680 .Text = blk: {
1681 const prev_node = root_node.decls.at(i - 1);1681 const prev_node = root_node.decls.at(i - 1);
1682 const prev_line_index = prev_node.lastToken().line;1682 const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, decl.firstToken());
1683 const this_line_index = decl.firstToken().line;1683 if (loc.line >= 2) {
1684 if (this_line_index - prev_line_index >= 2) {
1685 break :blk "\n\n";1684 break :blk "\n\n";
1686 }1685 }
1687 break :blk "\n";1686 break :blk "\n";
...@@ -1858,10 +1857,9 @@ pub const Parser = struct {...@@ -1858,10 +1857,9 @@ pub const Parser = struct {
1858 try stack.append(RenderState {1857 try stack.append(RenderState {
1859 .Text = blk: {1858 .Text = blk: {
1860 if (i != 0) {1859 if (i != 0) {
1861 const prev_statement_node = block.statements.items[i - 1];1860 const prev_node = block.statements.items[i - 1];
1862 const prev_line_index = prev_statement_node.lastToken().line;1861 const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, statement_node.firstToken());
1863 const this_line_index = statement_node.firstToken().line;1862 if (loc.line >= 2) {
1864 if (this_line_index - prev_line_index >= 2) {
1865 break :blk "\n\n";1863 break :blk "\n\n";
1866 }1864 }
1867 }1865 }
...@@ -2083,9 +2081,8 @@ pub const Parser = struct {...@@ -2083,9 +2081,8 @@ pub const Parser = struct {
2083 .Text = blk: {2081 .Text = blk: {
2084 if (i != 0) {2082 if (i != 0) {
2085 const prev_node = fields_and_decls[i - 1];2083 const prev_node = fields_and_decls[i - 1];
2086 const prev_line_index = prev_node.lastToken().line;2084 const loc = self.tokenizer.getTokenLocation(prev_node.lastToken().end, node.firstToken());
2087 const this_line_index = node.firstToken().line;2085 if (loc.line >= 2) {
2088 if (this_line_index - prev_line_index >= 2) {
2089 break :blk "\n\n";2086 break :blk "\n\n";
2090 }2087 }
2091 }2088 }
std/zig/tokenizer.zig+15-29
...@@ -5,8 +5,6 @@ pub const Token = struct {...@@ -5,8 +5,6 @@ pub const Token = struct {
5 id: Id,5 id: Id,
6 start: usize,6 start: usize,
7 end: usize,7 end: usize,
8 line: usize,
9 column: usize,
108
11 const KeywordId = struct {9 const KeywordId = struct {
12 bytes: []const u8,10 bytes: []const u8,
...@@ -180,28 +178,34 @@ pub const Token = struct {...@@ -180,28 +178,34 @@ pub const Token = struct {
180pub const Tokenizer = struct {178pub const Tokenizer = struct {
181 buffer: []const u8,179 buffer: []const u8,
182 index: usize,180 index: usize,
183 line: usize,
184 column: usize,
185 pending_invalid_token: ?Token,181 pending_invalid_token: ?Token,
186182
187 pub const LineLocation = struct {183 pub const Location = struct {
184 line: usize,
185 column: usize,
188 line_start: usize,186 line_start: usize,
189 line_end: usize,187 line_end: usize,
190 };188 };
191189
192 pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation {190 pub fn getTokenLocation(self: &Tokenizer, start_index: usize, token: &const Token) Location {
193 var loc = LineLocation {191 var loc = Location {
194 .line_start = 0,192 .line = 0,
193 .column = 0,
194 .line_start = start_index,
195 .line_end = self.buffer.len,195 .line_end = self.buffer.len,
196 };196 };
197 for (self.buffer) |c, i| {197 for (self.buffer[start_index..]) |c, i| {
198 if (i == token.start) {198 if (i + start_index == token.start) {
199 loc.line_end = i;199 loc.line_end = i + start_index;
200 while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}200 while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
201 return loc;201 return loc;
202 }202 }
203 if (c == '\n') {203 if (c == '\n') {
204 loc.line += 1;
205 loc.column = 0;
204 loc.line_start = i + 1;206 loc.line_start = i + 1;
207 } else {
208 loc.column += 1;
205 }209 }
206 }210 }
207 return loc;211 return loc;
...@@ -216,8 +220,6 @@ pub const Tokenizer = struct {...@@ -216,8 +220,6 @@ pub const Tokenizer = struct {
216 return Tokenizer {220 return Tokenizer {
217 .buffer = buffer,221 .buffer = buffer,
218 .index = 0,222 .index = 0,
219 .line = 0,
220 .column = 0,
221 .pending_invalid_token = null,223 .pending_invalid_token = null,
222 };224 };
223 }225 }
...@@ -277,8 +279,6 @@ pub const Tokenizer = struct {...@@ -277,8 +279,6 @@ pub const Tokenizer = struct {
277 .id = Token.Id.Eof,279 .id = Token.Id.Eof,
278 .start = self.index,280 .start = self.index,
279 .end = undefined,281 .end = undefined,
280 .line = self.line,
281 .column = self.column,
282 };282 };
283 while (self.index < self.buffer.len) : (self.index += 1) {283 while (self.index < self.buffer.len) : (self.index += 1) {
284 const c = self.buffer[self.index];284 const c = self.buffer[self.index];
...@@ -286,12 +286,9 @@ pub const Tokenizer = struct {...@@ -286,12 +286,9 @@ pub const Tokenizer = struct {
286 State.Start => switch (c) {286 State.Start => switch (c) {
287 ' ' => {287 ' ' => {
288 result.start = self.index + 1;288 result.start = self.index + 1;
289 result.column += 1;
290 },289 },
291 '\n' => {290 '\n' => {
292 result.start = self.index + 1;291 result.start = self.index + 1;
293 result.line += 1;
294 result.column = 0;
295 },292 },
296 'c' => {293 'c' => {
297 state = State.C;294 state = State.C;
...@@ -977,15 +974,6 @@ pub const Tokenizer = struct {...@@ -977,15 +974,6 @@ pub const Tokenizer = struct {
977 }974 }
978 }975 }
979976
980 for (self.buffer[start_index..self.index]) |c| {
981 if (c == '\n') {
982 self.line += 1;
983 self.column = 0;
984 } else {
985 self.column += 1;
986 }
987 }
988
989 if (result.id == Token.Id.Eof) {977 if (result.id == Token.Id.Eof) {
990 if (self.pending_invalid_token) |token| {978 if (self.pending_invalid_token) |token| {
991 self.pending_invalid_token = null;979 self.pending_invalid_token = null;
...@@ -1009,8 +997,6 @@ pub const Tokenizer = struct {...@@ -1009,8 +997,6 @@ pub const Tokenizer = struct {
1009 .id = Token.Id.Invalid,997 .id = Token.Id.Invalid,
1010 .start = self.index,998 .start = self.index,
1011 .end = self.index + invalid_length,999 .end = self.index + invalid_length,
1012 .line = self.line,
1013 .column = self.column,
1014 };1000 };
1015 }1001 }
10161002