authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 11:18:18+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 11:18:18+02:00
log22e38ffb54775c2e523665ac739f2a895d8757ad
tree79ab74c2127736ac330e01fb58221cd1bd756dd0
parenta2330d0ea366e0ced7fa917d3bf1ccd022e932c3

std.zig.tokenizer fixed tokens having wrong column and line

1 files changed, 12 insertions(+), 9 deletions(-)

std/zig/tokenizer.zig+12-9
......@@ -271,6 +271,7 @@ pub const Tokenizer = struct {
271271 self.pending_invalid_token = null;
272272 return token;
273273 }
274 const start_index = self.index;
274275 var state = State.Start;
275276 var result = Token {
276277 .id = Token.Id.Eof,
......@@ -279,7 +280,7 @@ pub const Tokenizer = struct {
279280 .line = self.line,
280281 .column = self.column,
281282 };
282 while (self.index < self.buffer.len) {
283 while (self.index < self.buffer.len) : (self.index += 1) {
283284 const c = self.buffer[self.index];
284285 switch (state) {
285286 State.Start => switch (c) {
......@@ -877,14 +878,6 @@ pub const Tokenizer = struct {
877878 else => break,
878879 },
879880 }
880
881 self.index += 1;
882 if (c == '\n') {
883 self.line += 1;
884 self.column = 0;
885 } else {
886 self.column += 1;
887 }
888881 } else if (self.index == self.buffer.len) {
889882 switch (state) {
890883 State.Start,
......@@ -983,6 +976,16 @@ pub const Tokenizer = struct {
983976 },
984977 }
985978 }
979
980 for (self.buffer[start_index..self.index]) |c| {
981 if (c == '\n') {
982 self.line += 1;
983 self.column = 0;
984 } else {
985 self.column += 1;
986 }
987 }
988
986989 if (result.id == Token.Id.Eof) {
987990 if (self.pending_invalid_token) |token| {
988991 self.pending_invalid_token = null;