authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-09-19 14:16:58+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-09-21 15:23:41-04:00
logced061fcbf6c00b91e93428456647b280ae58e1b
treeb31003cefa193a85fcd2299ead3bb3c320ba48d1
parentb759308fb30b130ea5d084c3814b1820a589ceee

stage2: update uses of DepTokenizer

2 files changed, 20 insertions(+), 36 deletions(-)

src-self-hosted/Cache.zig+17-21
...@@ -460,35 +460,31 @@ pub const CacheHash = struct {...@@ -460,35 +460,31 @@ pub const CacheHash = struct {
460 const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);460 const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
461 defer self.cache.gpa.free(dep_file_contents);461 defer self.cache.gpa.free(dep_file_contents);
462462
463 const DepTokenizer = @import("DepTokenizer.zig");463 var error_buf = std.ArrayList(u8).init(self.cache.gpa);
464 var it = DepTokenizer.init(self.cache.gpa, dep_file_contents);464 defer error_buf.deinit();
465 defer it.deinit();465
466 var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents };
466467
467 // Skip first token: target.468 // Skip first token: target.
468 {469 switch (it.next() orelse return) { // Empty dep file OK.
469 const opt_result = it.next() catch |err| switch (err) {470 .target, .target_must_resolve, .prereq => {},
470 error.OutOfMemory => return error.OutOfMemory,471 else => |err| {
471 error.InvalidInput => {472 try err.printError(error_buf.writer());
472 std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text });473 std.log.err("failed parsing {}: {}", .{ dep_file_basename, error_buf.items });
473 return error.InvalidDepFile;474 return error.InvalidDepFile;
474 },475 },
475 };
476 _ = opt_result orelse return; // Empty dep file OK.
477 }476 }
478 // Process 0+ preqreqs.477 // Process 0+ preqreqs.
479 // Clang is invoked in single-source mode so we never get more targets.478 // Clang is invoked in single-source mode so we never get more targets.
480 while (true) {479 while (true) {
481 const opt_result = it.next() catch |err| switch (err) {480 switch (it.next() orelse return) {
482 error.OutOfMemory => return error.OutOfMemory,481 .target, .target_must_resolve => return,
483 error.InvalidInput => {482 .prereq => |bytes| try self.addFilePost(bytes),
484 std.log.err("failed parsing {}: {}: {}", .{ dep_file_basename, @errorName(err), it.error_text });483 else => |err| {
484 try err.printError(error_buf.writer());
485 std.log.err("failed parsing {}: {}", .{ dep_file_basename, error_buf.items });
485 return error.InvalidDepFile;486 return error.InvalidDepFile;
486 },487 },
487 };
488 const result = opt_result orelse return;
489 switch (result.id) {
490 .target => return,
491 .prereq => try self.addFilePost(result.bytes),
492 }488 }
493 }489 }
494 }490 }
src-self-hosted/DepTokenizer.zig+3-15
...@@ -1,25 +1,13 @@...@@ -1,25 +1,13 @@
1const Tokenizer = @This();1const Tokenizer = @This();
22
3index: usize,3index: usize = 0,
4bytes: []const u8,4bytes: []const u8,
5state: State,5state: State = .lhs,
66
7const std = @import("std");7const std = @import("std");
8const testing = std.testing;8const testing = std.testing;
9const assert = std.debug.assert;9const assert = std.debug.assert;
1010
11pub fn init(allocator: *std.mem.Allocator, bytes: []const u8) Tokenizer {
12 return Tokenizer{
13 .index = 0,
14 .bytes = bytes,
15 .state = .lhs,
16 };
17}
18
19pub fn deinit(self: *Tokenizer) void {
20 self.arena.deinit();
21}
22
23pub fn next(self: *Tokenizer) ?Token {11pub fn next(self: *Tokenizer) ?Token {
24 var start = self.index;12 var start = self.index;
25 var must_resolve = false;13 var must_resolve = false;
...@@ -896,7 +884,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {...@@ -896,7 +884,7 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
896 const arena = &arena_allocator.allocator;884 const arena = &arena_allocator.allocator;
897 defer arena_allocator.deinit();885 defer arena_allocator.deinit();
898886
899 var it = Tokenizer.init(arena, input);887 var it: Tokenizer = .{ .bytes = input };
900 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);888 var buffer = try std.ArrayListSentineled(u8, 0).initSize(arena, 0);
901 var resolve_buf = std.ArrayList(u8).init(arena);889 var resolve_buf = std.ArrayList(u8).init(arena);
902 var i: usize = 0;890 var i: usize = 0;