authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-09-19 00:08:46+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 15:23:41-04:00
log302e1565237a0f54101e9c50af34039321dfbc29
tree4b21e77e1f98c5bbda0c32849b99e828c07d690c
parent2ef68631cb7045c277b348777b1a064845b95cd8

stage2: make DepTokenizer non-allocating

1 files changed, 168 insertions(+), 207 deletions(-)

src-self-hosted/DepTokenizer.zig+168-207
...@@ -1,9 +1,7 @@...@@ -1,9 +1,7 @@
1const Tokenizer = @This();1const Tokenizer = @This();
22
3arena: std.heap.ArenaAllocator,
4index: usize,3index: usize,
5bytes: []const u8,4bytes: []const u8,
6error_text: []const u8,
7state: State,5state: State,
86
9const std = @import("std");7const std = @import("std");
...@@ -12,11 +10,9 @@ const assert = std.debug.assert;...@@ -12,11 +10,9 @@ const assert = std.debug.assert;
1210
13pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer {11pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer {
14 return Tokenizer{12 return Tokenizer{
15 .arena = std.heap.ArenaAllocator.init(allocator),
16 .index = 0,13 .index = 0,
17 .bytes = bytes,14 .bytes = bytes,
18 .error_text = "",15 .state = .lhs,
19 .state = State{ .lhs = {} },
20 };16 };
21}17}
2218
...@@ -24,339 +20,306 @@ pub fn deinit(self: *Tokenizer) void {...@@ -24,339 +20,306 @@ pub fn deinit(self: *Tokenizer) void {
24 self.arena.deinit();20 self.arena.deinit();
25}21}
2622
27pub fn next(self: *Tokenizer) Error!?Token {23pub fn next(self: *Tokenizer) ?Token {
24 var start = self.index;
25 var must_resolve = false;
28 while (self.index < self.bytes.len) {26 while (self.index < self.bytes.len) {
29 const char = self.bytes[self.index];27 const char = self.bytes[self.index];
30 while (true) {
31 switch (self.state) {28 switch (self.state) {
32 .lhs => switch (char) {29 .lhs => switch (char) {
33 '\t', '\n', '\r', ' ' => {30 '\t', '\n', '\r', ' ' => {
34 // silently ignore whitespace31 // silently ignore whitespace
35 break; // advance32 self.index += 1;
36 },33 },
37 else => {34 else => {
38 self.state = State{ .target = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };35 start = self.index;
36 self.state = .target;
39 },37 },
40 },38 },
41 .target => |*target| switch (char) {39 .target => switch (char) {
42 '\t', '\n', '\r', ' ' => {40 '\t', '\n', '\r', ' ' => {
43 return self.errorIllegalChar(self.index, char, "invalid target", .{});41 return errorIllegalChar(.invalid_target, self.index, char);
44 },42 },
45 '$' => {43 '$' => {
46 self.state = State{ .target_dollar_sign = target.* };44 self.state = .target_dollar_sign;
47 break; // advance45 self.index += 1;
48 },46 },
49 '\\' => {47 '\\' => {
50 self.state = State{ .target_reverse_solidus = target.* };48 self.state = .target_reverse_solidus;
51 break; // advance49 self.index += 1;
52 },50 },
53 ':' => {51 ':' => {
54 self.state = State{ .target_colon = target.* };52 self.state = .target_colon;
55 break; // advance53 self.index += 1;
56 },54 },
57 else => {55 else => {
58 try target.append(char);56 self.index += 1;
59 break; // advance
60 },57 },
61 },58 },
62 .target_reverse_solidus => |*target| switch (char) {59 .target_reverse_solidus => switch (char) {
63 '\t', '\n', '\r' => {60 '\t', '\n', '\r' => {
64 return self.errorIllegalChar(self.index, char, "bad target escape", .{});61 return errorIllegalChar(.bad_target_escape, self.index, char);
65 },62 },
66 ' ', '#', '\\' => {63 ' ', '#', '\\' => {
67 try target.append(char);64 must_resolve = true;
68 self.state = State{ .target = target.* };65 self.state = .target;
69 break; // advance66 self.index += 1;
70 },67 },
71 '$' => {68 '$' => {
72 try target.appendSlice(self.bytes[self.index - 1 .. self.index]);69 self.state = .target_dollar_sign;
73 self.state = State{ .target_dollar_sign = target.* };70 self.index += 1;
74 break; // advance
75 },71 },
76 else => {72 else => {
77 try target.appendSlice(self.bytes[self.index - 1 .. self.index + 1]);73 self.state = .target;
78 self.state = State{ .target = target.* };74 self.index += 1;
79 break; // advance
80 },75 },
81 },76 },
82 .target_dollar_sign => |*target| switch (char) {77 .target_dollar_sign => switch (char) {
83 '$' => {78 '$' => {
84 try target.append(char);79 must_resolve = true;
85 self.state = State{ .target = target.* };80 self.state = .target;
86 break; // advance81 self.index += 1;
87 },82 },
88 else => {83 else => {
89 return self.errorIllegalChar(self.index, char, "expecting '$'", .{});84 return errorIllegalChar(.expected_dollar_sign, self.index, char);
90 },85 },
91 },86 },
92 .target_colon => |*target| switch (char) {87 .target_colon => switch (char) {
93 '\n', '\r' => {88 '\n', '\r' => {
94 const bytes = target.span();89 const bytes = self.bytes[start..self.index - 1];
95 if (bytes.len != 0) {90 if (bytes.len != 0) {
96 self.state = State{ .lhs = {} };91 self.state = .lhs;
97 return Token{ .id = .target, .bytes = bytes };92 return finishTarget(must_resolve, bytes);
98 }93 }
99 // silently ignore null target94 // silently ignore null target
100 self.state = State{ .lhs = {} };95 self.state = .lhs;
101 continue;
102 },96 },
103 '\\' => {97 '\\' => {
104 self.state = State{ .target_colon_reverse_solidus = target.* };98 self.state = .target_colon_reverse_solidus;
105 break; // advance99 self.index += 1;
106 },100 },
107 else => {101 else => {
108 const bytes = target.span();102 const bytes = self.bytes[start..self.index - 1];
109 if (bytes.len != 0) {103 if (bytes.len != 0) {
110 self.state = State{ .rhs = {} };104 self.state = .rhs;
111 return Token{ .id = .target, .bytes = bytes };105 return finishTarget(must_resolve, bytes);
112 }106 }
113 // silently ignore null target107 // silently ignore null target
114 self.state = State{ .lhs = {} };108 self.state = .lhs;
115 continue;
116 },109 },
117 },110 },
118 .target_colon_reverse_solidus => |*target| switch (char) {111 .target_colon_reverse_solidus => switch (char) {
119 '\n', '\r' => {112 '\n', '\r' => {
120 const bytes = target.span();113 const bytes = self.bytes[start .. self.index - 2];
121 if (bytes.len != 0) {114 if (bytes.len != 0) {
122 self.state = State{ .lhs = {} };115 self.state = .lhs;
123 return Token{ .id = .target, .bytes = bytes };116 return finishTarget(must_resolve, bytes);
124 }117 }
125 // silently ignore null target118 // silently ignore null target
126 self.state = State{ .lhs = {} };119 self.state = .lhs;
127 continue;
128 },120 },
129 else => {121 else => {
130 try target.appendSlice(self.bytes[self.index - 2 .. self.index + 1]);122 self.state = .target;
131 self.state = State{ .target = target.* };
132 break;
133 },123 },
134 },124 },
135 .rhs => switch (char) {125 .rhs => switch (char) {
136 '\t', ' ' => {126 '\t', ' ' => {
137 // silently ignore horizontal whitespace127 // silently ignore horizontal whitespace
138 break; // advance128 self.index += 1;
139 },129 },
140 '\n', '\r' => {130 '\n', '\r' => {
141 self.state = State{ .lhs = {} };131 self.state = .lhs;
142 continue;
143 },132 },
144 '\\' => {133 '\\' => {
145 self.state = State{ .rhs_continuation = {} };134 self.state = .rhs_continuation;
146 break; // advance135 self.index += 1;
147 },136 },
148 '"' => {137 '"' => {
149 self.state = State{ .prereq_quote = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };138 self.state = .prereq_quote;
150 break; // advance139 self.index += 1;
140 start = self.index;
151 },141 },
152 else => {142 else => {
153 self.state = State{ .prereq = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };143 start = self.index;
144 self.state = .prereq;
154 },145 },
155 },146 },
156 .rhs_continuation => switch (char) {147 .rhs_continuation => switch (char) {
157 '\n' => {148 '\n' => {
158 self.state = State{ .rhs = {} };149 self.state = .rhs;
159 break; // advance150 self.index += 1;
160 },151 },
161 '\r' => {152 '\r' => {
162 self.state = State{ .rhs_continuation_linefeed = {} };153 self.state = .rhs_continuation_linefeed;
163 break; // advance154 self.index += 1;
164 },155 },
165 else => {156 else => {
166 return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});157 return errorIllegalChar(.continuation_eol, self.index, char);
167 },158 },
168 },159 },
169 .rhs_continuation_linefeed => switch (char) {160 .rhs_continuation_linefeed => switch (char) {
170 '\n' => {161 '\n' => {
171 self.state = State{ .rhs = {} };162 self.state = .rhs;
172 break; // advance163 self.index += 1;
173 },164 },
174 else => {165 else => {
175 return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});166 return errorIllegalChar(.continuation_eol, self.index, char);
176 },167 },
177 },168 },
178 .prereq_quote => |*prereq| switch (char) {169 .prereq_quote => switch (char) {
179 '"' => {170 '"' => {
180 const bytes = prereq.span();
181 self.index += 1;171 self.index += 1;
182 self.state = State{ .rhs = {} };172 self.state = .rhs;
183 return Token{ .id = .prereq, .bytes = bytes };173 return Token{ .prereq = self.bytes[start .. self.index - 1] };
184 },174 },
185 else => {175 else => {
186 try prereq.append(char);176 self.index += 1;
187 break; // advance
188 },177 },
189 },178 },
190 .prereq => |*prereq| switch (char) {179 .prereq => switch (char) {
191 '\t', ' ' => {180 '\t', ' ' => {
192 const bytes = prereq.span();181 self.state = .rhs;
193 self.state = State{ .rhs = {} };182 return Token{ .prereq = self.bytes[start..self.index] };
194 return Token{ .id = .prereq, .bytes = bytes };
195 },183 },
196 '\n', '\r' => {184 '\n', '\r' => {
197 const bytes = prereq.span();185 self.state = .lhs;
198 self.state = State{ .lhs = {} };186 return Token{ .prereq = self.bytes[start..self.index] };
199 return Token{ .id = .prereq, .bytes = bytes };
200 },187 },
201 '\\' => {188 '\\' => {
202 self.state = State{ .prereq_continuation = prereq.* };189 self.state = .prereq_continuation;
203 break; // advance190 self.index += 1;
204 },191 },
205 else => {192 else => {
206 try prereq.append(char);193 self.index += 1;
207 break; // advance
208 },194 },
209 },195 },
210 .prereq_continuation => |*prereq| switch (char) {196 .prereq_continuation => switch (char) {
211 '\n' => {197 '\n' => {
212 const bytes = prereq.span();
213 self.index += 1;198 self.index += 1;
214 self.state = State{ .rhs = {} };199 self.state = .rhs;
215 return Token{ .id = .prereq, .bytes = bytes };200 return Token{ .prereq = self.bytes[start .. self.index - 2] };
216 },201 },
217 '\r' => {202 '\r' => {
218 self.state = State{ .prereq_continuation_linefeed = prereq.* };203 self.state = .prereq_continuation_linefeed;
219 break; // advance204 self.index += 1;
220 },205 },
221 else => {206 else => {
222 // not continuation207 // not continuation
223 try prereq.appendSlice(self.bytes[self.index - 1 .. self.index + 1]);208 self.state = .prereq;
224 self.state = State{ .prereq = prereq.* };209 self.index += 1;
225 break; // advance
226 },210 },
227 },211 },
228 .prereq_continuation_linefeed => |prereq| switch (char) {212 .prereq_continuation_linefeed => switch (char) {
229 '\n' => {213 '\n' => {
230 const bytes = prereq.span();
231 self.index += 1;214 self.index += 1;
232 self.state = State{ .rhs = {} };215 self.state = .rhs;
233 return Token{ .id = .prereq, .bytes = bytes };216 return Token{ .prereq = self.bytes[start .. self.index - 1] };
234 },217 },
235 else => {218 else => {
236 return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});219 return errorIllegalChar(.continuation_eol, self.index, char);
237 },220 },
238 },221 },
239 }222 }
240 }223 } else {
241 self.index += 1;
242 }
243
244 // eof, handle maybe incomplete token
245 if (self.index == 0) return null;
246 const idx = self.index - 1;
247 switch (self.state) {224 switch (self.state) {
248 .lhs,225 .lhs,
249 .rhs,226 .rhs,
250 .rhs_continuation,227 .rhs_continuation,
251 .rhs_continuation_linefeed,228 .rhs_continuation_linefeed,
252 => {},229 => return null,
253 .target => |target| {230 .target => {
254 return self.errorPosition(idx, target.span(), "incomplete target", .{});231 return Token{ .incomplete_target = self.bytes[start..] };
255 },232 },
256 .target_reverse_solidus,233 .target_reverse_solidus,
257 .target_dollar_sign,234 .target_dollar_sign,
258 => {235 => {
259 const index = self.index - 1;236 const idx = self.index - 1;
260 return self.errorIllegalChar(idx, self.bytes[idx], "incomplete escape", .{});237 return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
261 },238 },
262 .target_colon => |target| {239 .target_colon => {
263 const bytes = target.span();240 const bytes = self.bytes[start.. self.index - 1];
264 if (bytes.len != 0) {241 if (bytes.len != 0) {
265 self.index += 1;242 self.index += 1;
266 self.state = State{ .rhs = {} };243 self.state = .rhs;
267 return Token{ .id = .target, .bytes = bytes };244 return finishTarget(must_resolve, bytes);
268 }245 }
269 // silently ignore null target246 // silently ignore null target
270 self.state = State{ .lhs = {} };247 self.state = .lhs;
248 return null;
271 },249 },
272 .target_colon_reverse_solidus => |target| {250 .target_colon_reverse_solidus => {
273 const bytes = target.span();251 const bytes = self.bytes[start..self.index - 2];
274 if (bytes.len != 0) {252 if (bytes.len != 0) {
275 self.index += 1;253 self.index += 1;
276 self.state = State{ .rhs = {} };254 self.state = .rhs;
277 return Token{ .id = .target, .bytes = bytes };255 return finishTarget(must_resolve, bytes);
278 }256 }
279 // silently ignore null target257 // silently ignore null target
280 self.state = State{ .lhs = {} };258 self.state = .lhs;
259 return null;
281 },260 },
282 .prereq_quote => |prereq| {261 .prereq_quote => {
283 return self.errorPosition(idx, prereq.span(), "incomplete quoted prerequisite", .{});262 return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] };
284 },263 },
285 .prereq => |prereq| {264 .prereq => {
286 const bytes = prereq.span();265 self.state = .lhs;
287 self.state = State{ .lhs = {} };266 return Token{ .prereq = self.bytes[start..] };
288 return Token{ .id = .prereq, .bytes = bytes };
289 },267 },
290 .prereq_continuation => |prereq| {268 .prereq_continuation => {
291 const bytes = prereq.span();269 self.state = .lhs;
292 self.state = State{ .lhs = {} };270 return Token{ .prereq = self.bytes[start.. self.index - 1] };
293 return Token{ .id = .prereq, .bytes = bytes };
294 },271 },
295 .prereq_continuation_linefeed => |prereq| {272 .prereq_continuation_linefeed => {
296 const bytes = prereq.span();273 self.state = .lhs;
297 self.state = State{ .lhs = {} };274 return Token{ .prereq = self.bytes[start.. self.index - 2] };
298 return Token{ .id = .prereq, .bytes = bytes };
299 },275 },
300 }276 }
301 return null;277}
278 unreachable;
302}279}
303280
304fn errorf(self: *Tokenizer, comptime fmt: []const u8, args: anytype) Error {281fn errorIllegalChar(comptime id: @TagType(Token), index: usize, char: u8) Token {
305 self.error_text = try std.fmt.allocPrintZ(&self.arena.allocator, fmt, args);282 return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
306 return Error.InvalidInput;
307}283}
308284
309fn errorPosition(self: *Tokenizer, position: usize, bytes: []const u8, comptime fmt: []const u8, args: anytype) Error {285fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
310 var buffer = std.ArrayList(u8).init(&self.arena.allocator);286 return if (must_resolve)
311 try buffer.outStream().print(fmt, args);287 .{ .target_must_resolve = bytes }
312 try buffer.appendSlice(" '");288 else
313 const out = buffer.writer();289 .{ .target = bytes };
314 try printCharValues(out, bytes);290}
315 try buffer.appendSlice("'");
316 try buffer.outStream().print(" at position {}", .{position - (bytes.len - 1)});
317 try buffer.append(0);
318 self.error_text = buffer.items[0 .. buffer.items.len - 1 :0];
319 return Error.InvalidInput;
320}
321
322fn errorIllegalChar(self: *Tokenizer, position: usize, char: u8, comptime fmt: []const u8, args: anytype) Error {
323 var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0);
324 try buffer.appendSlice("illegal char ");
325 try printUnderstandableChar(&buffer, char);
326 try buffer.outStream().print(" at position {}", .{position});
327 if (fmt.len != 0) try buffer.outStream().print(": " ++ fmt, args);
328 self.error_text = buffer.span();
329 return Error.InvalidInput;
330}
331
332const Error = error{
333 OutOfMemory,
334 InvalidInput,
335};
336291
337const State = union(enum) {292const State = enum {
338 lhs: void,293 lhs,
339 target: std.ArrayListSentineled(u8, 0),294 target,
340 target_reverse_solidus: std.ArrayListSentineled(u8, 0),295 target_reverse_solidus,
341 target_dollar_sign: std.ArrayListSentineled(u8, 0),296 target_dollar_sign,
342 target_colon: std.ArrayListSentineled(u8, 0),297 target_colon,
343 target_colon_reverse_solidus: std.ArrayListSentineled(u8, 0),298 target_colon_reverse_solidus,
344 rhs: void,299 rhs,
345 rhs_continuation: void,300 rhs_continuation,
346 rhs_continuation_linefeed: void,301 rhs_continuation_linefeed,
347 prereq_quote: std.ArrayListSentineled(u8, 0),302 prereq_quote,
348 prereq: std.ArrayListSentineled(u8, 0),303 prereq,
349 prereq_continuation: std.ArrayListSentineled(u8, 0),304 prereq_continuation,
350 prereq_continuation_linefeed: std.ArrayListSentineled(u8, 0),305 prereq_continuation_linefeed,
351};306};
352307
353pub const Token = struct {308pub const Token = union(enum) {
354 id: ID,309 target: []const u8,
355 bytes: []const u8,310 target_must_resolve: []const u8,
356311 prereq: []const u8,
357 pub const ID = enum {312 incomplete_quoted_prerequisite: []const u8,
358 target,313 incomplete_target: []const u8,
359 prereq,314 invalid_target: IndexAndChar,
315 bad_target_escape: IndexAndChar,
316 expected_dollar_sign: IndexAndChar,
317 continuation_eol: IndexAndChar,
318 incomplete_escape: IndexAndChar,
319
320 pub const IndexAndChar = struct {
321 index: usize,
322 char: u8,
360 };323 };
361};324};
362325
...@@ -845,26 +808,24 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {...@@ -845,26 +808,24 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
845 var it = Tokenizer.init(arena, input);808 var it = Tokenizer.init(arena, input);
846 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);809 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);
847 var i: usize = 0;810 var i: usize = 0;
848 while (true) {811 while (it.next()) |token| {
849 const r = it.next() catch |err| {
850 switch (err) {
851 Tokenizer.Error.InvalidInput => {
852 if (i != 0) try buffer.appendSlice("\n");
853 try buffer.appendSlice("ERROR: ");
854 try buffer.appendSlice(it.error_text);
855 },
856 else => return err,
857 }
858 break;
859 };
860 const token = r orelse break;
861 if (i != 0) try buffer.appendSlice("\n");812 if (i != 0) try buffer.appendSlice("\n");
862 try buffer.appendSlice(@tagName(token.id));813 switch (token) {
814 .target, .prereq => |bytes| {
815 try buffer.appendSlice(@tagName(token));
863 try buffer.appendSlice(" = {");816 try buffer.appendSlice(" = {");
864 for (token.bytes) |b| {817 for (bytes) |b| {
865 try buffer.append(printable_char_tab[b]);818 try buffer.append(printable_char_tab[b]);
866 }819 }
867 try buffer.appendSlice("}");820 try buffer.appendSlice("}");
821 },
822 .target_must_resolve => {
823 @panic("TODO");
824 },
825 else => {
826 @panic("TODO");
827 },
828 }
868 i += 1;829 i += 1;
869 }830 }
870 const got: []const u8 = buffer.span();831 const got: []const u8 = buffer.span();