authorgravatar for rpkak@noreply.codeberg.orgrpkak <rpkak@noreply.codeberg.org> 2025-03-14 12:18:10+01:00
committergravatar for alex@alexrp.comAlex Rønne Petersen <alex@alexrp.com> 2025-03-24 15:31:03+01:00
log9f8d938d3825bd5d4d2d7f76907075d13b04d8e1
treed8479ec4f95f55c258b2a8e6c1e70d543bb06997
parent677b2d62e5af4d7a97883994b8d8c45a06535ea0

DepTokenizer: allow space between target and colon

1 files changed, 60 insertions(+), 4 deletions(-)

lib/std/Build/Cache/DepTokenizer.zig+60-4
...@@ -25,7 +25,7 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -25,7 +25,7 @@ pub fn next(self: *Tokenizer) ?Token {
25 },25 },
26 },26 },
27 .target => switch (char) {27 .target => switch (char) {
28 '\t', '\n', '\r', ' ' => {28 '\n', '\r' => {
29 return errorIllegalChar(.invalid_target, self.index, char);29 return errorIllegalChar(.invalid_target, self.index, char);
30 },30 },
31 '$' => {31 '$' => {
...@@ -40,6 +40,15 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -40,6 +40,15 @@ pub fn next(self: *Tokenizer) ?Token {
40 self.state = .target_colon;40 self.state = .target_colon;
41 self.index += 1;41 self.index += 1;
42 },42 },
43 '\t', ' ' => {
44 self.state = .target_space;
45
46 const bytes = self.bytes[start..self.index];
47 std.debug.assert(bytes.len != 0);
48 self.index += 1;
49
50 return finishTarget(must_resolve, bytes);
51 },
43 else => {52 else => {
44 self.index += 1;53 self.index += 1;
45 },54 },
...@@ -110,6 +119,19 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -110,6 +119,19 @@ pub fn next(self: *Tokenizer) ?Token {
110 self.state = .target;119 self.state = .target;
111 },120 },
112 },121 },
122 .target_space => switch (char) {
123 '\t', ' ' => {
124 // silently ignore additional horizontal whitespace
125 self.index += 1;
126 },
127 ':' => {
128 self.state = .rhs;
129 self.index += 1;
130 },
131 else => {
132 return errorIllegalChar(.expected_colon, self.index, char);
133 },
134 },
113 .rhs => switch (char) {135 .rhs => switch (char) {
114 '\t', ' ' => {136 '\t', ' ' => {
115 // silently ignore horizontal whitespace137 // silently ignore horizontal whitespace
...@@ -256,6 +278,10 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -256,6 +278,10 @@ pub fn next(self: *Tokenizer) ?Token {
256 self.state = .lhs;278 self.state = .lhs;
257 return null;279 return null;
258 },280 },
281 .target_space => {
282 const idx = self.index - 1;
283 return errorIllegalChar(.expected_colon, idx, self.bytes[idx]);
284 },
259 .prereq_quote => {285 .prereq_quote => {
260 return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);286 return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
261 },287 },
...@@ -299,6 +325,7 @@ const State = enum {...@@ -299,6 +325,7 @@ const State = enum {
299 target_dollar_sign,325 target_dollar_sign,
300 target_colon,326 target_colon,
301 target_colon_reverse_solidus,327 target_colon_reverse_solidus,
328 target_space,
302 rhs,329 rhs,
303 rhs_continuation,330 rhs_continuation,
304 rhs_continuation_linefeed,331 rhs_continuation_linefeed,
...@@ -322,6 +349,7 @@ pub const Token = union(enum) {...@@ -322,6 +349,7 @@ pub const Token = union(enum) {
322 expected_dollar_sign: IndexAndChar,349 expected_dollar_sign: IndexAndChar,
323 continuation_eol: IndexAndChar,350 continuation_eol: IndexAndChar,
324 incomplete_escape: IndexAndChar,351 incomplete_escape: IndexAndChar,
352 expected_colon: IndexAndChar,
325353
326 pub const IndexAndChar = struct {354 pub const IndexAndChar = struct {
327 index: usize,355 index: usize,
...@@ -420,6 +448,7 @@ pub const Token = union(enum) {...@@ -420,6 +448,7 @@ pub const Token = union(enum) {
420 .expected_dollar_sign,448 .expected_dollar_sign,
421 .continuation_eol,449 .continuation_eol,
422 .incomplete_escape,450 .incomplete_escape,
451 .expected_colon,
423 => |index_and_char| {452 => |index_and_char| {
424 try writer.writeAll("illegal char ");453 try writer.writeAll("illegal char ");
425 try printUnderstandableChar(writer, index_and_char.char);454 try printUnderstandableChar(writer, index_and_char.char);
...@@ -438,6 +467,7 @@ pub const Token = union(enum) {...@@ -438,6 +467,7 @@ pub const Token = union(enum) {
438 .expected_dollar_sign => "expecting '$'",467 .expected_dollar_sign => "expecting '$'",
439 .continuation_eol => "continuation expecting end-of-line",468 .continuation_eol => "continuation expecting end-of-line",
440 .incomplete_escape => "incomplete escape",469 .incomplete_escape => "incomplete escape",
470 .expected_colon => "expecting ':'",
441 };471 };
442 }472 }
443};473};
...@@ -545,6 +575,16 @@ test "empty target linefeeds + hspace + continuations" {...@@ -545,6 +575,16 @@ test "empty target linefeeds + hspace + continuations" {
545 , expect);575 , expect);
546}576}
547577
578test "empty target + hspace + colon" {
579 const expect = "target = {foo.o}";
580
581 try depTokenizer("foo.o :", expect);
582 try depTokenizer("foo.o\t\t\t:", expect);
583 try depTokenizer("foo.o \t \t :", expect);
584 try depTokenizer("\r\nfoo.o :", expect);
585 try depTokenizer(" foo.o :", expect);
586}
587
548test "prereq" {588test "prereq" {
549 const expect =589 const expect =
550 \\target = {foo.o}590 \\target = {foo.o}
...@@ -923,9 +963,6 @@ test "error illegal char at position - expecting dollar_sign" {...@@ -923,9 +963,6 @@ test "error illegal char at position - expecting dollar_sign" {
923}963}
924964
925test "error illegal char at position - invalid target" {965test "error illegal char at position - invalid target" {
926 try depTokenizer("foo\t.o",
927 \\ERROR: illegal char \x09 at position 3: invalid target
928 );
929 try depTokenizer("foo\n.o",966 try depTokenizer("foo\n.o",
930 \\ERROR: illegal char \x0A at position 3: invalid target967 \\ERROR: illegal char \x0A at position 3: invalid target
931 );968 );
...@@ -963,6 +1000,25 @@ test "error prereq - continuation expecting end-of-line" {...@@ -963,6 +1000,25 @@ test "error prereq - continuation expecting end-of-line" {
963 );1000 );
964}1001}
9651002
1003test "error illegal char at position - expecting colon" {
1004 try depTokenizer("foo\t.o:",
1005 \\target = {foo}
1006 \\ERROR: illegal char '.' at position 4: expecting ':'
1007 );
1008 try depTokenizer("foo .o:",
1009 \\target = {foo}
1010 \\ERROR: illegal char '.' at position 4: expecting ':'
1011 );
1012 try depTokenizer("foo \n.o:",
1013 \\target = {foo}
1014 \\ERROR: illegal char \x0A at position 4: expecting ':'
1015 );
1016 try depTokenizer("foo.o\t\n:",
1017 \\target = {foo.o}
1018 \\ERROR: illegal char \x0A at position 6: expecting ':'
1019 );
1020}
1021
966// - tokenize input, emit textual representation, and compare to expect1022// - tokenize input, emit textual representation, and compare to expect
967fn depTokenizer(input: []const u8, expect: []const u8) !void {1023fn depTokenizer(input: []const u8, expect: []const u8) !void {
968 var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);1024 var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);