authorgravatar for github@magicmoremagic.comBen Crist <github@magicmoremagic.com> 2024-06-06 13:40:10-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-06-06 18:40:10+00:00
loga9e9c9965daa8a6d26fd1d4baf4f31f558b0d5fa
tree4f8972c047e97a092004c25ce9f6ed05b473156b
parent63754916c598a3be41171abb708dc6434f93c63e
signaturebadge-check Signed by PGP key B5690EEEBB952194

Build system: Support Windows depfiles with unquoted, backslash escaped spaces (#20100)

2 files changed, 140 insertions(+), 44 deletions(-)

lib/std/Build/Cache.zig+11-2
......@@ -860,14 +860,23 @@ pub const Manifest = struct {
860860
861861 var it: DepTokenizer = .{ .bytes = dep_file_contents };
862862
863 while (true) {
864 switch (it.next() orelse return) {
863 while (it.next()) |token| {
864 switch (token) {
865865 // We don't care about targets, we only want the prereqs
866866 // Clang is invoked in single-source mode but other programs may not
867867 .target, .target_must_resolve => {},
868868 .prereq => |file_path| if (self.manifest_file == null) {
869869 _ = try self.addFile(file_path, null);
870870 } else try self.addFilePost(file_path),
871 .prereq_must_resolve => {
872 var resolve_buf = std.ArrayList(u8).init(self.cache.gpa);
873 defer resolve_buf.deinit();
874
875 try token.resolve(resolve_buf.writer());
876 if (self.manifest_file == null) {
877 _ = try self.addFile(resolve_buf.items, null);
878 } else try self.addFilePost(resolve_buf.items);
879 },
871880 else => |err| {
872881 try err.printError(error_buf.writer());
873882 log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
lib/std/Build/Cache/DepTokenizer.zig+129-42
......@@ -158,7 +158,7 @@ pub fn next(self: *Tokenizer) ?Token {
158158 '"' => {
159159 self.index += 1;
160160 self.state = .rhs;
161 return Token{ .prereq = self.bytes[start .. self.index - 1] };
161 return finishPrereq(must_resolve, self.bytes[start .. self.index - 1]);
162162 },
163163 else => {
164164 self.index += 1;
......@@ -167,11 +167,11 @@ pub fn next(self: *Tokenizer) ?Token {
167167 .prereq => switch (char) {
168168 '\t', ' ' => {
169169 self.state = .rhs;
170 return Token{ .prereq = self.bytes[start..self.index] };
170 return finishPrereq(must_resolve, self.bytes[start..self.index]);
171171 },
172172 '\n', '\r' => {
173173 self.state = .lhs;
174 return Token{ .prereq = self.bytes[start..self.index] };
174 return finishPrereq(must_resolve, self.bytes[start..self.index]);
175175 },
176176 '\\' => {
177177 self.state = .prereq_continuation;
......@@ -185,12 +185,22 @@ pub fn next(self: *Tokenizer) ?Token {
185185 '\n' => {
186186 self.index += 1;
187187 self.state = .rhs;
188 return Token{ .prereq = self.bytes[start .. self.index - 2] };
188 return finishPrereq(must_resolve, self.bytes[start .. self.index - 2]);
189189 },
190190 '\r' => {
191191 self.state = .prereq_continuation_linefeed;
192192 self.index += 1;
193193 },
194 '\\' => {
195 // The previous \ wasn't a continuation, but this one might be.
196 self.index += 1;
197 },
198 ' ' => {
199 // not continuation, but escaped space must be resolved
200 must_resolve = true;
201 self.state = .prereq;
202 self.index += 1;
203 },
194204 else => {
195205 // not continuation
196206 self.state = .prereq;
......@@ -201,7 +211,7 @@ pub fn next(self: *Tokenizer) ?Token {
201211 '\n' => {
202212 self.index += 1;
203213 self.state = .rhs;
204 return Token{ .prereq = self.bytes[start .. self.index - 1] };
214 return finishPrereq(must_resolve, self.bytes[start .. self.index - 3]);
205215 },
206216 else => {
207217 return errorIllegalChar(.continuation_eol, self.index, char);
......@@ -251,15 +261,15 @@ pub fn next(self: *Tokenizer) ?Token {
251261 },
252262 .prereq => {
253263 self.state = .lhs;
254 return Token{ .prereq = self.bytes[start..] };
264 return finishPrereq(must_resolve, self.bytes[start..]);
255265 },
256266 .prereq_continuation => {
257267 self.state = .lhs;
258 return Token{ .prereq = self.bytes[start .. self.index - 1] };
268 return finishPrereq(must_resolve, self.bytes[start .. self.index - 1]);
259269 },
260270 .prereq_continuation_linefeed => {
261271 self.state = .lhs;
262 return Token{ .prereq = self.bytes[start .. self.index - 2] };
272 return finishPrereq(must_resolve, self.bytes[start .. self.index - 2]);
263273 },
264274 }
265275 }
......@@ -278,6 +288,10 @@ fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
278288 return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes };
279289}
280290
291fn finishPrereq(must_resolve: bool, bytes: []const u8) Token {
292 return if (must_resolve) .{ .prereq_must_resolve = bytes } else .{ .prereq = bytes };
293}
294
281295const State = enum {
282296 lhs,
283297 target,
......@@ -298,6 +312,7 @@ pub const Token = union(enum) {
298312 target: []const u8,
299313 target_must_resolve: []const u8,
300314 prereq: []const u8,
315 prereq_must_resolve: []const u8,
301316
302317 incomplete_quoted_prerequisite: IndexAndBytes,
303318 incomplete_target: IndexAndBytes,
......@@ -318,48 +333,76 @@ pub const Token = union(enum) {
318333 bytes: []const u8,
319334 };
320335
321 /// Resolve escapes in target. Only valid with .target_must_resolve.
336 /// Resolve escapes in target or prereq. Only valid with .target_must_resolve or .prereq_must_resolve.
322337 pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
323 const bytes = self.target_must_resolve; // resolve called on incorrect token
324
325 var state: enum { start, escape, dollar } = .start;
326 for (bytes) |c| {
327 switch (state) {
328 .start => {
329 switch (c) {
330 '\\' => state = .escape,
331 '$' => state = .dollar,
332 else => try writer.writeByte(c),
333 }
334 },
335 .escape => {
336 switch (c) {
337 ' ', '#', '\\' => {},
338 '$' => {
339 try writer.writeByte('\\');
340 state = .dollar;
341 continue;
338 switch (self) {
339 .target_must_resolve => |bytes| {
340 var state: enum { start, escape, dollar } = .start;
341 for (bytes) |c| {
342 switch (state) {
343 .start => {
344 switch (c) {
345 '\\' => state = .escape,
346 '$' => state = .dollar,
347 else => try writer.writeByte(c),
348 }
349 },
350 .escape => {
351 switch (c) {
352 ' ', '#', '\\' => {},
353 '$' => {
354 try writer.writeByte('\\');
355 state = .dollar;
356 continue;
357 },
358 else => try writer.writeByte('\\'),
359 }
360 try writer.writeByte(c);
361 state = .start;
362 },
363 .dollar => {
364 try writer.writeByte('$');
365 switch (c) {
366 '$' => {},
367 else => try writer.writeByte(c),
368 }
369 state = .start;
342370 },
343 else => try writer.writeByte('\\'),
344371 }
345 try writer.writeByte(c);
346 state = .start;
347 },
348 .dollar => {
349 try writer.writeByte('$');
350 switch (c) {
351 '$' => {},
352 else => try writer.writeByte(c),
372 }
373 },
374 .prereq_must_resolve => |bytes| {
375 var state: enum { start, escape } = .start;
376 for (bytes) |c| {
377 switch (state) {
378 .start => {
379 switch (c) {
380 '\\' => state = .escape,
381 else => try writer.writeByte(c),
382 }
383 },
384 .escape => {
385 switch (c) {
386 ' ' => {},
387 '\\' => {
388 try writer.writeByte(c);
389 continue;
390 },
391 else => try writer.writeByte('\\'),
392 }
393 try writer.writeByte(c);
394 state = .start;
395 },
353396 }
354 state = .start;
355 },
356 }
397 }
398 },
399 else => unreachable,
357400 }
358401 }
359402
360403 pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
361404 switch (self) {
362 .target, .target_must_resolve, .prereq => unreachable, // not an error
405 .target, .target_must_resolve, .prereq, .prereq_must_resolve => unreachable, // not an error
363406 .incomplete_quoted_prerequisite,
364407 .incomplete_target,
365408 => |index_and_bytes| {
......@@ -387,7 +430,7 @@ pub const Token = union(enum) {
387430
388431 fn errStr(self: Token) []const u8 {
389432 return switch (self) {
390 .target, .target_must_resolve, .prereq => unreachable, // not an error
433 .target, .target_must_resolve, .prereq, .prereq_must_resolve => unreachable, // not an error
391434 .incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
392435 .incomplete_target => "incomplete target",
393436 .invalid_target => "invalid target",
......@@ -538,6 +581,15 @@ test "prereq continuation" {
538581 , expect);
539582}
540583
584test "prereq continuation (CRLF)" {
585 const expect =
586 \\target = {foo.o}
587 \\prereq = {foo.h}
588 \\prereq = {bar.h}
589 ;
590 try depTokenizer("foo.o: foo.h\\\r\nbar.h", expect);
591}
592
541593test "multiple prereqs" {
542594 const expect =
543595 \\target = {foo.o}
......@@ -728,6 +780,32 @@ test "windows funky targets" {
728780 );
729781}
730782
783test "windows funky prereqs" {
784 // Note we don't support unquoted escaped spaces at the very beginning of a relative path
785 // e.g. `\ SpaceAtTheBeginning.c`
786 // This typically wouldn't be seen in the wild, since depfiles usually use absolute paths
787 // and supporting it would degrade error messages for cases where it was meant to be a
788 // continuation, but the line ending is missing.
789 try depTokenizer(
790 \\cimport.o: \
791 \\ trailingbackslash\\
792 \\ C:\Users\John\ Smith\AppData\Local\zig\p\1220d14057af1a9d6dde4643293527bd5ee5099517d655251a066666a4320737ea7c\cimport.c \
793 \\ somedir\\ a.c\
794 \\ somedir/\ a.c\
795 \\ somedir\\ \ \ b.c\
796 \\ somedir\\ \\ \c.c\
797 \\
798 ,
799 \\target = {cimport.o}
800 \\prereq = {trailingbackslash\}
801 \\prereq = {C:\Users\John Smith\AppData\Local\zig\p\1220d14057af1a9d6dde4643293527bd5ee5099517d655251a066666a4320737ea7c\cimport.c}
802 \\prereq = {somedir\ a.c}
803 \\prereq = {somedir/ a.c}
804 \\prereq = {somedir\ b.c}
805 \\prereq = {somedir\ \ \c.c}
806 );
807}
808
731809test "windows drive and forward slashes" {
732810 try depTokenizer(
733811 \\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \
......@@ -915,6 +993,15 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
915993 resolve_buf.items.len = 0;
916994 try buffer.appendSlice("}");
917995 },
996 .prereq_must_resolve => {
997 try buffer.appendSlice("prereq = {");
998 try token.resolve(resolve_buf.writer());
999 for (resolve_buf.items) |b| {
1000 try buffer.append(printable_char_tab[b]);
1001 }
1002 resolve_buf.items.len = 0;
1003 try buffer.appendSlice("}");
1004 },
9181005 else => {
9191006 try buffer.appendSlice("ERROR: ");
9201007 try token.printError(buffer.writer());