authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-09-19 14:16:58+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 15:23:41-04:00
logced061fcbf6c00b91e93428456647b280ae58e1b
treeb31003cefa193a85fcd2299ead3bb3c320ba48d1
parentb759308fb30b130ea5d084c3814b1820a589ceee

stage2: update uses of DepTokenizer

2 files changed, 20 insertions(+), 36 deletions(-)

src-self-hosted/Cache.zig+17-21
......@@ -460,35 +460,31 @@ pub const CacheHash = struct {
460460 const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
461461 defer self.cache.gpa.free(dep_file_contents);
462462
463 const DepTokenizer = @import("DepTokenizer.zig");
464 var it = DepTokenizer.init(self.cache.gpa, dep_file_contents);
465 defer it.deinit();
463 var error_buf = std.ArrayList(u8).init(self.cache.gpa);
464 defer error_buf.deinit();
465
466 var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents };
466467
467468 // Skip first token: target.
468 {
469 const opt_result = it.next() catch |err| switch (err) {
470 error.OutOfMemory => return error.OutOfMemory,
471 error.InvalidInput => {
472 std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text });
473 return error.InvalidDepFile;
474 },
475 };
476 _ = opt_result orelse return; // Empty dep file OK.
469 switch (it.next() orelse return) { // Empty dep file OK.
470 .target, .target_must_resolve, .prereq => {},
471 else => |err| {
472 try err.printError(error_buf.writer());
473 std.log.err("failed parsing {}: {}", .{ dep_file_basename, error_buf.items });
474 return error.InvalidDepFile;
475 },
477476 }
478477 // Process 0+ preqreqs.
479478 // Clang is invoked in single-source mode so we never get more targets.
480479 while (true) {
481 const opt_result = it.next() catch |err| switch (err) {
482 error.OutOfMemory => return error.OutOfMemory,
483 error.InvalidInput => {
484 std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text });
480 switch (it.next() orelse return) {
481 .target, .target_must_resolve => return,
482 .prereq => |bytes| try self.addFilePost(bytes),
483 else => |err| {
484 try err.printError(error_buf.writer());
485 std.log.err("failed parsing {}: {}", .{ dep_file_basename, error_buf.items });
485486 return error.InvalidDepFile;
486487 },
487 };
488 const result = opt_result orelse return;
489 switch (result.id) {
490 .target => return,
491 .prereq => try self.addFilePost(result.bytes),
492488 }
493489 }
494490 }
src-self-hosted/DepTokenizer.zig+3-15
......@@ -1,25 +1,13 @@
11const Tokenizer = @This();
22
3index: usize,
3index: usize = 0,
44bytes: []const u8,
5state: State,
5state: State = .lhs,
66
77const std = @import("std");
88const testing = std.testing;
99const assert = std.debug.assert;
1010
11pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer {
12 return Tokenizer{
13 .index = 0,
14 .bytes = bytes,
15 .state = .lhs,
16 };
17}
18
19pub fn deinit(self: *Tokenizer) void {
20 self.arena.deinit();
21}
22
2311pub fn next(self: *Tokenizer) ?Token {
2412 var start = self.index;
2513 var must_resolve = false;
......@@ -896,7 +884,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
896884 const arena = &arena_allocator.allocator;
897885 defer arena_allocator.deinit();
898886
899 var it = Tokenizer.init(arena, input);
887 var it: Tokenizer = .{ .bytes = input };
900888 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);
901889 var resolve_buf = std.ArrayList(u8).init(arena);
902890 var i: usize = 0;