authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-09-19 00:08:46+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 15:23:41-04:00
log302e1565237a0f54101e9c50af34039321dfbc29
tree4b21e77e1f98c5bbda0c32849b99e828c07d690c
parent2ef68631cb7045c277b348777b1a064845b95cd8

stage2: make DepTokenizer non-allocating

1 files changed, 168 insertions(+), 207 deletions(-)

src-self-hosted/DepTokenizer.zig+168-207
......@@ -1,9 +1,7 @@
11const Tokenizer = @This();
22
3arena: std.heap.ArenaAllocator,
43index: usize,
54bytes: []const u8,
6error_text: []const u8,
75state: State,
86
97const std = @import("std");
......@@ -12,11 +10,9 @@ const assert = std.debug.assert;
1210
1311pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer {
1412 return Tokenizer{
15 .arena = std.heap.ArenaAllocator.init(allocator),
1613 .index = 0,
1714 .bytes = bytes,
18 .error_text = "",
19 .state = State{ .lhs = {} },
15 .state = .lhs,
2016 };
2117}
2218
......@@ -24,339 +20,306 @@ pub fn deinit(self: *Tokenizer) void {
2420 self.arena.deinit();
2521}
2622
27pub fn next(self: *Tokenizer) Error!?Token {
23pub fn next(self: *Tokenizer) ?Token {
24 var start = self.index;
25 var must_resolve = false;
2826 while (self.index < self.bytes.len) {
2927 const char = self.bytes[self.index];
30 while (true) {
3128 switch (self.state) {
3229 .lhs => switch (char) {
3330 '\t', '\n', '\r', ' ' => {
3431 // silently ignore whitespace
35 break; // advance
32 self.index += 1;
3633 },
3734 else => {
38 self.state = State{ .target = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
35 start = self.index;
36 self.state = .target;
3937 },
4038 },
41 .target => |*target| switch (char) {
39 .target => switch (char) {
4240 '\t', '\n', '\r', ' ' => {
43 return self.errorIllegalChar(self.index, char, "invalid target", .{});
41 return errorIllegalChar(.invalid_target, self.index, char);
4442 },
4543 '$' => {
46 self.state = State{ .target_dollar_sign = target.* };
47 break; // advance
44 self.state = .target_dollar_sign;
45 self.index += 1;
4846 },
4947 '\\' => {
50 self.state = State{ .target_reverse_solidus = target.* };
51 break; // advance
48 self.state = .target_reverse_solidus;
49 self.index += 1;
5250 },
5351 ':' => {
54 self.state = State{ .target_colon = target.* };
55 break; // advance
52 self.state = .target_colon;
53 self.index += 1;
5654 },
5755 else => {
58 try target.append(char);
59 break; // advance
56 self.index += 1;
6057 },
6158 },
62 .target_reverse_solidus => |*target| switch (char) {
59 .target_reverse_solidus => switch (char) {
6360 '\t', '\n', '\r' => {
64 return self.errorIllegalChar(self.index, char, "bad target escape", .{});
61 return errorIllegalChar(.bad_target_escape, self.index, char);
6562 },
6663 ' ', '#', '\\' => {
67 try target.append(char);
68 self.state = State{ .target = target.* };
69 break; // advance
64 must_resolve = true;
65 self.state = .target;
66 self.index += 1;
7067 },
7168 '$' => {
72 try target.appendSlice(self.bytes[self.index - 1 .. self.index]);
73 self.state = State{ .target_dollar_sign = target.* };
74 break; // advance
69 self.state = .target_dollar_sign;
70 self.index += 1;
7571 },
7672 else => {
77 try target.appendSlice(self.bytes[self.index - 1 .. self.index + 1]);
78 self.state = State{ .target = target.* };
79 break; // advance
73 self.state = .target;
74 self.index += 1;
8075 },
8176 },
82 .target_dollar_sign => |*target| switch (char) {
77 .target_dollar_sign => switch (char) {
8378 '$' => {
84 try target.append(char);
85 self.state = State{ .target = target.* };
86 break; // advance
79 must_resolve = true;
80 self.state = .target;
81 self.index += 1;
8782 },
8883 else => {
89 return self.errorIllegalChar(self.index, char, "expecting '$'", .{});
84 return errorIllegalChar(.expected_dollar_sign, self.index, char);
9085 },
9186 },
92 .target_colon => |*target| switch (char) {
87 .target_colon => switch (char) {
9388 '\n', '\r' => {
94 const bytes = target.span();
89 const bytes = self.bytes[start..self.index - 1];
9590 if (bytes.len != 0) {
96 self.state = State{ .lhs = {} };
97 return Token{ .id = .target, .bytes = bytes };
91 self.state = .lhs;
92 return finishTarget(must_resolve, bytes);
9893 }
9994 // silently ignore null target
100 self.state = State{ .lhs = {} };
101 continue;
95 self.state = .lhs;
10296 },
10397 '\\' => {
104 self.state = State{ .target_colon_reverse_solidus = target.* };
105 break; // advance
98 self.state = .target_colon_reverse_solidus;
99 self.index += 1;
106100 },
107101 else => {
108 const bytes = target.span();
102 const bytes = self.bytes[start..self.index - 1];
109103 if (bytes.len != 0) {
110 self.state = State{ .rhs = {} };
111 return Token{ .id = .target, .bytes = bytes };
104 self.state = .rhs;
105 return finishTarget(must_resolve, bytes);
112106 }
113107 // silently ignore null target
114 self.state = State{ .lhs = {} };
115 continue;
108 self.state = .lhs;
116109 },
117110 },
118 .target_colon_reverse_solidus => |*target| switch (char) {
111 .target_colon_reverse_solidus => switch (char) {
119112 '\n', '\r' => {
120 const bytes = target.span();
113 const bytes = self.bytes[start .. self.index - 2];
121114 if (bytes.len != 0) {
122 self.state = State{ .lhs = {} };
123 return Token{ .id = .target, .bytes = bytes };
115 self.state = .lhs;
116 return finishTarget(must_resolve, bytes);
124117 }
125118 // silently ignore null target
126 self.state = State{ .lhs = {} };
127 continue;
119 self.state = .lhs;
128120 },
129121 else => {
130 try target.appendSlice(self.bytes[self.index - 2 .. self.index + 1]);
131 self.state = State{ .target = target.* };
132 break;
122 self.state = .target;
133123 },
134124 },
135125 .rhs => switch (char) {
136126 '\t', ' ' => {
137127 // silently ignore horizontal whitespace
138 break; // advance
128 self.index += 1;
139129 },
140130 '\n', '\r' => {
141 self.state = State{ .lhs = {} };
142 continue;
131 self.state = .lhs;
143132 },
144133 '\\' => {
145 self.state = State{ .rhs_continuation = {} };
146 break; // advance
134 self.state = .rhs_continuation;
135 self.index += 1;
147136 },
148137 '"' => {
149 self.state = State{ .prereq_quote = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
150 break; // advance
138 self.state = .prereq_quote;
139 self.index += 1;
140 start = self.index;
151141 },
152142 else => {
153 self.state = State{ .prereq = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0) };
143 start = self.index;
144 self.state = .prereq;
154145 },
155146 },
156147 .rhs_continuation => switch (char) {
157148 '\n' => {
158 self.state = State{ .rhs = {} };
159 break; // advance
149 self.state = .rhs;
150 self.index += 1;
160151 },
161152 '\r' => {
162 self.state = State{ .rhs_continuation_linefeed = {} };
163 break; // advance
153 self.state = .rhs_continuation_linefeed;
154 self.index += 1;
164155 },
165156 else => {
166 return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
157 return errorIllegalChar(.continuation_eol, self.index, char);
167158 },
168159 },
169160 .rhs_continuation_linefeed => switch (char) {
170161 '\n' => {
171 self.state = State{ .rhs = {} };
172 break; // advance
162 self.state = .rhs;
163 self.index += 1;
173164 },
174165 else => {
175 return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
166 return errorIllegalChar(.continuation_eol, self.index, char);
176167 },
177168 },
178 .prereq_quote => |*prereq| switch (char) {
169 .prereq_quote => switch (char) {
179170 '"' => {
180 const bytes = prereq.span();
181171 self.index += 1;
182 self.state = State{ .rhs = {} };
183 return Token{ .id = .prereq, .bytes = bytes };
172 self.state = .rhs;
173 return Token{ .prereq = self.bytes[start .. self.index - 1] };
184174 },
185175 else => {
186 try prereq.append(char);
187 break; // advance
176 self.index += 1;
188177 },
189178 },
190 .prereq => |*prereq| switch (char) {
179 .prereq => switch (char) {
191180 '\t', ' ' => {
192 const bytes = prereq.span();
193 self.state = State{ .rhs = {} };
194 return Token{ .id = .prereq, .bytes = bytes };
181 self.state = .rhs;
182 return Token{ .prereq = self.bytes[start..self.index] };
195183 },
196184 '\n', '\r' => {
197 const bytes = prereq.span();
198 self.state = State{ .lhs = {} };
199 return Token{ .id = .prereq, .bytes = bytes };
185 self.state = .lhs;
186 return Token{ .prereq = self.bytes[start..self.index] };
200187 },
201188 '\\' => {
202 self.state = State{ .prereq_continuation = prereq.* };
203 break; // advance
189 self.state = .prereq_continuation;
190 self.index += 1;
204191 },
205192 else => {
206 try prereq.append(char);
207 break; // advance
193 self.index += 1;
208194 },
209195 },
210 .prereq_continuation => |*prereq| switch (char) {
196 .prereq_continuation => switch (char) {
211197 '\n' => {
212 const bytes = prereq.span();
213198 self.index += 1;
214 self.state = State{ .rhs = {} };
215 return Token{ .id = .prereq, .bytes = bytes };
199 self.state = .rhs;
200 return Token{ .prereq = self.bytes[start .. self.index - 2] };
216201 },
217202 '\r' => {
218 self.state = State{ .prereq_continuation_linefeed = prereq.* };
219 break; // advance
203 self.state = .prereq_continuation_linefeed;
204 self.index += 1;
220205 },
221206 else => {
222207 // not continuation
223 try prereq.appendSlice(self.bytes[self.index - 1 .. self.index + 1]);
224 self.state = State{ .prereq = prereq.* };
225 break; // advance
208 self.state = .prereq;
209 self.index += 1;
226210 },
227211 },
228 .prereq_continuation_linefeed => |prereq| switch (char) {
212 .prereq_continuation_linefeed => switch (char) {
229213 '\n' => {
230 const bytes = prereq.span();
231214 self.index += 1;
232 self.state = State{ .rhs = {} };
233 return Token{ .id = .prereq, .bytes = bytes };
215 self.state = .rhs;
216 return Token{ .prereq = self.bytes[start .. self.index - 1] };
234217 },
235218 else => {
236 return self.errorIllegalChar(self.index, char, "continuation expecting end-of-line", .{});
219 return errorIllegalChar(.continuation_eol, self.index, char);
237220 },
238221 },
239222 }
240 }
241 self.index += 1;
242 }
243
244 // eof, handle maybe incomplete token
245 if (self.index == 0) return null;
246 const idx = self.index - 1;
223 } else {
247224 switch (self.state) {
248225 .lhs,
249226 .rhs,
250227 .rhs_continuation,
251228 .rhs_continuation_linefeed,
252 => {},
253 .target => |target| {
254 return self.errorPosition(idx, target.span(), "incomplete target", .{});
229 => return null,
230 .target => {
231 return Token{ .incomplete_target = self.bytes[start..] };
255232 },
256233 .target_reverse_solidus,
257234 .target_dollar_sign,
258235 => {
259 const index = self.index - 1;
260 return self.errorIllegalChar(idx, self.bytes[idx], "incomplete escape", .{});
236 const idx = self.index - 1;
237 return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
261238 },
262 .target_colon => |target| {
263 const bytes = target.span();
239 .target_colon => {
240 const bytes = self.bytes[start.. self.index - 1];
264241 if (bytes.len != 0) {
265242 self.index += 1;
266 self.state = State{ .rhs = {} };
267 return Token{ .id = .target, .bytes = bytes };
243 self.state = .rhs;
244 return finishTarget(must_resolve, bytes);
268245 }
269246 // silently ignore null target
270 self.state = State{ .lhs = {} };
247 self.state = .lhs;
248 return null;
271249 },
272 .target_colon_reverse_solidus => |target| {
273 const bytes = target.span();
250 .target_colon_reverse_solidus => {
251 const bytes = self.bytes[start..self.index - 2];
274252 if (bytes.len != 0) {
275253 self.index += 1;
276 self.state = State{ .rhs = {} };
277 return Token{ .id = .target, .bytes = bytes };
254 self.state = .rhs;
255 return finishTarget(must_resolve, bytes);
278256 }
279257 // silently ignore null target
280 self.state = State{ .lhs = {} };
258 self.state = .lhs;
259 return null;
281260 },
282 .prereq_quote => |prereq| {
283 return self.errorPosition(idx, prereq.span(), "incomplete quoted prerequisite", .{});
261 .prereq_quote => {
262 return Token{ .incomplete_quoted_prerequisite = self.bytes[start..] };
284263 },
285 .prereq => |prereq| {
286 const bytes = prereq.span();
287 self.state = State{ .lhs = {} };
288 return Token{ .id = .prereq, .bytes = bytes };
264 .prereq => {
265 self.state = .lhs;
266 return Token{ .prereq = self.bytes[start..] };
289267 },
290 .prereq_continuation => |prereq| {
291 const bytes = prereq.span();
292 self.state = State{ .lhs = {} };
293 return Token{ .id = .prereq, .bytes = bytes };
268 .prereq_continuation => {
269 self.state = .lhs;
270 return Token{ .prereq = self.bytes[start.. self.index - 1] };
294271 },
295 .prereq_continuation_linefeed => |prereq| {
296 const bytes = prereq.span();
297 self.state = State{ .lhs = {} };
298 return Token{ .id = .prereq, .bytes = bytes };
272 .prereq_continuation_linefeed => {
273 self.state = .lhs;
274 return Token{ .prereq = self.bytes[start.. self.index - 2] };
299275 },
300276 }
301 return null;
277}
278 unreachable;
302279}
303280
304fn errorf(self: *Tokenizer, comptime fmt: []const u8, args: anytype) Error {
305 self.error_text = try std.fmt.allocPrintZ(&self.arena.allocator, fmt, args);
306 return Error.InvalidInput;
281fn errorIllegalChar(comptime id: @TagType(Token), index: usize, char: u8) Token {
282 return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
307283}
308284
309fn errorPosition(self: *Tokenizer, position: usize, bytes: []const u8, comptime fmt: []const u8, args: anytype) Error {
310 var buffer = std.ArrayList(u8).init(&self.arena.allocator);
311 try buffer.outStream().print(fmt, args);
312 try buffer.appendSlice(" '");
313 const out = buffer.writer();
314 try printCharValues(out, bytes);
315 try buffer.appendSlice("'");
316 try buffer.outStream().print(" at position {}", .{position - (bytes.len - 1)});
317 try buffer.append(0);
318 self.error_text = buffer.items[0 .. buffer.items.len - 1 :0];
319 return Error.InvalidInput;
320}
321
322fn errorIllegalChar(self: *Tokenizer, position: usize, char: u8, comptime fmt: []const u8, args: anytype) Error {
323 var buffer = try std.ArrayListSentineled(u8, 0).initSize(&self.arena.allocator, 0);
324 try buffer.appendSlice("illegal char ");
325 try printUnderstandableChar(&buffer, char);
326 try buffer.outStream().print(" at position {}", .{position});
327 if (fmt.len != 0) try buffer.outStream().print(": " ++ fmt, args);
328 self.error_text = buffer.span();
329 return Error.InvalidInput;
330}
331
332const Error = error{
333 OutOfMemory,
334 InvalidInput,
335};
285fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
286 return if (must_resolve)
287 .{ .target_must_resolve = bytes }
288 else
289 .{ .target = bytes };
290}
336291
337const State = union(enum) {
338 lhs: void,
339 target: std.ArrayListSentineled(u8, 0),
340 target_reverse_solidus: std.ArrayListSentineled(u8, 0),
341 target_dollar_sign: std.ArrayListSentineled(u8, 0),
342 target_colon: std.ArrayListSentineled(u8, 0),
343 target_colon_reverse_solidus: std.ArrayListSentineled(u8, 0),
344 rhs: void,
345 rhs_continuation: void,
346 rhs_continuation_linefeed: void,
347 prereq_quote: std.ArrayListSentineled(u8, 0),
348 prereq: std.ArrayListSentineled(u8, 0),
349 prereq_continuation: std.ArrayListSentineled(u8, 0),
350 prereq_continuation_linefeed: std.ArrayListSentineled(u8, 0),
292const State = enum {
293 lhs,
294 target,
295 target_reverse_solidus,
296 target_dollar_sign,
297 target_colon,
298 target_colon_reverse_solidus,
299 rhs,
300 rhs_continuation,
301 rhs_continuation_linefeed,
302 prereq_quote,
303 prereq,
304 prereq_continuation,
305 prereq_continuation_linefeed,
351306};
352307
353pub const Token = struct {
354 id: ID,
355 bytes: []const u8,
356
357 pub const ID = enum {
358 target,
359 prereq,
308pub const Token = union(enum) {
309 target: []const u8,
310 target_must_resolve: []const u8,
311 prereq: []const u8,
312 incomplete_quoted_prerequisite: []const u8,
313 incomplete_target: []const u8,
314 invalid_target: IndexAndChar,
315 bad_target_escape: IndexAndChar,
316 expected_dollar_sign: IndexAndChar,
317 continuation_eol: IndexAndChar,
318 incomplete_escape: IndexAndChar,
319
320 pub const IndexAndChar = struct {
321 index: usize,
322 char: u8,
360323 };
361324};
362325
......@@ -845,26 +808,24 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
845808 var it = Tokenizer.init(arena, input);
846809 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);
847810 var i: usize = 0;
848 while (true) {
849 const r = it.next() catch |err| {
850 switch (err) {
851 Tokenizer.Error.InvalidInput => {
852 if (i != 0) try buffer.appendSlice("\n");
853 try buffer.appendSlice("ERROR: ");
854 try buffer.appendSlice(it.error_text);
855 },
856 else => return err,
857 }
858 break;
859 };
860 const token = r orelse break;
811 while (it.next()) |token| {
861812 if (i != 0) try buffer.appendSlice("\n");
862 try buffer.appendSlice(@tagName(token.id));
813 switch (token) {
814 .target, .prereq => |bytes| {
815 try buffer.appendSlice(@tagName(token));
863816 try buffer.appendSlice(" = {");
864 for (token.bytes) |b| {
817 for (bytes) |b| {
865818 try buffer.append(printable_char_tab[b]);
866819 }
867820 try buffer.appendSlice("}");
821 },
822 .target_must_resolve => {
823 @panic("TODO");
824 },
825 else => {
826 @panic("TODO");
827 },
828 }
868829 i += 1;
869830 }
870831 const got: []const u8 = buffer.span();