authorgravatar for github@magicmoremagic.comBen Crist <github@magicmoremagic.com> 2024-06-06 13:40:10-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2024-06-06 18:40:10+00:00
loga9e9c9965daa8a6d26fd1d4baf4f31f558b0d5fa
tree4f8972c047e97a092004c25ce9f6ed05b473156b
parent63754916c598a3be41171abb708dc6434f93c63e
signaturebadge-check Signed by PGP key B5690EEEBB952194

Build system: Support Windows depfiles with unquoted, backslash escaped spaces (#20100)

2 files changed, 140 insertions(+), 44 deletions(-)

lib/std/Build/Cache.zig+11-2
...@@ -860,14 +860,23 @@ pub const Manifest = struct {...@@ -860,14 +860,23 @@ pub const Manifest = struct {
860860
861 var it: DepTokenizer = .{ .bytes = dep_file_contents };861 var it: DepTokenizer = .{ .bytes = dep_file_contents };
862862
863 while (true) {863 while (it.next()) |token| {
864 switch (it.next() orelse return) {864 switch (token) {
865 // We don't care about targets, we only want the prereqs865 // We don't care about targets, we only want the prereqs
866 // Clang is invoked in single-source mode but other programs may not866 // Clang is invoked in single-source mode but other programs may not
867 .target, .target_must_resolve => {},867 .target, .target_must_resolve => {},
868 .prereq => |file_path| if (self.manifest_file == null) {868 .prereq => |file_path| if (self.manifest_file == null) {
869 _ = try self.addFile(file_path, null);869 _ = try self.addFile(file_path, null);
870 } else try self.addFilePost(file_path),870 } else try self.addFilePost(file_path),
871 .prereq_must_resolve => {
872 var resolve_buf = std.ArrayList(u8).init(self.cache.gpa);
873 defer resolve_buf.deinit();
874
875 try token.resolve(resolve_buf.writer());
876 if (self.manifest_file == null) {
877 _ = try self.addFile(resolve_buf.items, null);
878 } else try self.addFilePost(resolve_buf.items);
879 },
871 else => |err| {880 else => |err| {
872 try err.printError(error_buf.writer());881 try err.printError(error_buf.writer());
873 log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });882 log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
lib/std/Build/Cache/DepTokenizer.zig+129-42
...@@ -158,7 +158,7 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -158,7 +158,7 @@ pub fn next(self: *Tokenizer) ?Token {
158 '"' => {158 '"' => {
159 self.index += 1;159 self.index += 1;
160 self.state = .rhs;160 self.state = .rhs;
161 return Token{ .prereq = self.bytes[start .. self.index - 1] };161 return finishPrereq(must_resolve, self.bytes[start .. self.index - 1]);
162 },162 },
163 else => {163 else => {
164 self.index += 1;164 self.index += 1;
...@@ -167,11 +167,11 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -167,11 +167,11 @@ pub fn next(self: *Tokenizer) ?Token {
167 .prereq => switch (char) {167 .prereq => switch (char) {
168 '\t', ' ' => {168 '\t', ' ' => {
169 self.state = .rhs;169 self.state = .rhs;
170 return Token{ .prereq = self.bytes[start..self.index] };170 return finishPrereq(must_resolve, self.bytes[start..self.index]);
171 },171 },
172 '\n', '\r' => {172 '\n', '\r' => {
173 self.state = .lhs;173 self.state = .lhs;
174 return Token{ .prereq = self.bytes[start..self.index] };174 return finishPrereq(must_resolve, self.bytes[start..self.index]);
175 },175 },
176 '\\' => {176 '\\' => {
177 self.state = .prereq_continuation;177 self.state = .prereq_continuation;
...@@ -185,12 +185,22 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -185,12 +185,22 @@ pub fn next(self: *Tokenizer) ?Token {
185 '\n' => {185 '\n' => {
186 self.index += 1;186 self.index += 1;
187 self.state = .rhs;187 self.state = .rhs;
188 return Token{ .prereq = self.bytes[start .. self.index - 2] };188 return finishPrereq(must_resolve, self.bytes[start .. self.index - 2]);
189 },189 },
190 '\r' => {190 '\r' => {
191 self.state = .prereq_continuation_linefeed;191 self.state = .prereq_continuation_linefeed;
192 self.index += 1;192 self.index += 1;
193 },193 },
194 '\\' => {
195 // The previous \ wasn't a continuation, but this one might be.
196 self.index += 1;
197 },
198 ' ' => {
199 // not continuation, but escaped space must be resolved
200 must_resolve = true;
201 self.state = .prereq;
202 self.index += 1;
203 },
194 else => {204 else => {
195 // not continuation205 // not continuation
196 self.state = .prereq;206 self.state = .prereq;
...@@ -201,7 +211,7 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -201,7 +211,7 @@ pub fn next(self: *Tokenizer) ?Token {
201 '\n' => {211 '\n' => {
202 self.index += 1;212 self.index += 1;
203 self.state = .rhs;213 self.state = .rhs;
204 return Token{ .prereq = self.bytes[start .. self.index - 1] };214 return finishPrereq(must_resolve, self.bytes[start .. self.index - 3]);
205 },215 },
206 else => {216 else => {
207 return errorIllegalChar(.continuation_eol, self.index, char);217 return errorIllegalChar(.continuation_eol, self.index, char);
...@@ -251,15 +261,15 @@ pub fn next(self: *Tokenizer) ?Token {...@@ -251,15 +261,15 @@ pub fn next(self: *Tokenizer) ?Token {
251 },261 },
252 .prereq => {262 .prereq => {
253 self.state = .lhs;263 self.state = .lhs;
254 return Token{ .prereq = self.bytes[start..] };264 return finishPrereq(must_resolve, self.bytes[start..]);
255 },265 },
256 .prereq_continuation => {266 .prereq_continuation => {
257 self.state = .lhs;267 self.state = .lhs;
258 return Token{ .prereq = self.bytes[start .. self.index - 1] };268 return finishPrereq(must_resolve, self.bytes[start .. self.index - 1]);
259 },269 },
260 .prereq_continuation_linefeed => {270 .prereq_continuation_linefeed => {
261 self.state = .lhs;271 self.state = .lhs;
262 return Token{ .prereq = self.bytes[start .. self.index - 2] };272 return finishPrereq(must_resolve, self.bytes[start .. self.index - 2]);
263 },273 },
264 }274 }
265 }275 }
...@@ -278,6 +288,10 @@ fn finishTarget(must_resolve: bool, bytes: []const u8) Token {...@@ -278,6 +288,10 @@ fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
278 return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes };288 return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes };
279}289}
280290
291fn finishPrereq(must_resolve: bool, bytes: []const u8) Token {
292 return if (must_resolve) .{ .prereq_must_resolve = bytes } else .{ .prereq = bytes };
293}
294
281const State = enum {295const State = enum {
282 lhs,296 lhs,
283 target,297 target,
...@@ -298,6 +312,7 @@ pub const Token = union(enum) {...@@ -298,6 +312,7 @@ pub const Token = union(enum) {
298 target: []const u8,312 target: []const u8,
299 target_must_resolve: []const u8,313 target_must_resolve: []const u8,
300 prereq: []const u8,314 prereq: []const u8,
315 prereq_must_resolve: []const u8,
301316
302 incomplete_quoted_prerequisite: IndexAndBytes,317 incomplete_quoted_prerequisite: IndexAndBytes,
303 incomplete_target: IndexAndBytes,318 incomplete_target: IndexAndBytes,
...@@ -318,48 +333,76 @@ pub const Token = union(enum) {...@@ -318,48 +333,76 @@ pub const Token = union(enum) {
318 bytes: []const u8,333 bytes: []const u8,
319 };334 };
320335
321 /// Resolve escapes in target. Only valid with .target_must_resolve.336 /// Resolve escapes in target or prereq. Only valid with .target_must_resolve or .prereq_must_resolve.
322 pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {337 pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
323 const bytes = self.target_must_resolve; // resolve called on incorrect token338 switch (self) {
324339 .target_must_resolve => |bytes| {
325 var state: enum { start, escape, dollar } = .start;340 var state: enum { start, escape, dollar } = .start;
326 for (bytes) |c| {341 for (bytes) |c| {
327 switch (state) {342 switch (state) {
328 .start => {343 .start => {
329 switch (c) {344 switch (c) {
330 '\\' => state = .escape,345 '\\' => state = .escape,
331 '$' => state = .dollar,346 '$' => state = .dollar,
332 else => try writer.writeByte(c),347 else => try writer.writeByte(c),
333 }348 }
334 },349 },
335 .escape => {350 .escape => {
336 switch (c) {351 switch (c) {
337 ' ', '#', '\\' => {},352 ' ', '#', '\\' => {},
338 '$' => {353 '$' => {
339 try writer.writeByte('\\');354 try writer.writeByte('\\');
340 state = .dollar;355 state = .dollar;
341 continue;356 continue;
357 },
358 else => try writer.writeByte('\\'),
359 }
360 try writer.writeByte(c);
361 state = .start;
362 },
363 .dollar => {
364 try writer.writeByte('$');
365 switch (c) {
366 '$' => {},
367 else => try writer.writeByte(c),
368 }
369 state = .start;
342 },370 },
343 else => try writer.writeByte('\\'),
344 }371 }
345 try writer.writeByte(c);372 }
346 state = .start;373 },
347 },374 .prereq_must_resolve => |bytes| {
348 .dollar => {375 var state: enum { start, escape } = .start;
349 try writer.writeByte('$');376 for (bytes) |c| {
350 switch (c) {377 switch (state) {
351 '$' => {},378 .start => {
352 else => try writer.writeByte(c),379 switch (c) {
380 '\\' => state = .escape,
381 else => try writer.writeByte(c),
382 }
383 },
384 .escape => {
385 switch (c) {
386 ' ' => {},
387 '\\' => {
388 try writer.writeByte(c);
389 continue;
390 },
391 else => try writer.writeByte('\\'),
392 }
393 try writer.writeByte(c);
394 state = .start;
395 },
353 }396 }
354 state = .start;397 }
355 },398 },
356 }399 else => unreachable,
357 }400 }
358 }401 }
359402
360 pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {403 pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
361 switch (self) {404 switch (self) {
362 .target, .target_must_resolve, .prereq => unreachable, // not an error405 .target, .target_must_resolve, .prereq, .prereq_must_resolve => unreachable, // not an error
363 .incomplete_quoted_prerequisite,406 .incomplete_quoted_prerequisite,
364 .incomplete_target,407 .incomplete_target,
365 => |index_and_bytes| {408 => |index_and_bytes| {
...@@ -387,7 +430,7 @@ pub const Token = union(enum) {...@@ -387,7 +430,7 @@ pub const Token = union(enum) {
387430
388 fn errStr(self: Token) []const u8 {431 fn errStr(self: Token) []const u8 {
389 return switch (self) {432 return switch (self) {
390 .target, .target_must_resolve, .prereq => unreachable, // not an error433 .target, .target_must_resolve, .prereq, .prereq_must_resolve => unreachable, // not an error
391 .incomplete_quoted_prerequisite => "incomplete quoted prerequisite",434 .incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
392 .incomplete_target => "incomplete target",435 .incomplete_target => "incomplete target",
393 .invalid_target => "invalid target",436 .invalid_target => "invalid target",
...@@ -538,6 +581,15 @@ test "prereq continuation" {...@@ -538,6 +581,15 @@ test "prereq continuation" {
538 , expect);581 , expect);
539}582}
540583
584test "prereq continuation (CRLF)" {
585 const expect =
586 \\target = {foo.o}
587 \\prereq = {foo.h}
588 \\prereq = {bar.h}
589 ;
590 try depTokenizer("foo.o: foo.h\\\r\nbar.h", expect);
591}
592
541test "multiple prereqs" {593test "multiple prereqs" {
542 const expect =594 const expect =
543 \\target = {foo.o}595 \\target = {foo.o}
...@@ -728,6 +780,32 @@ test "windows funky targets" {...@@ -728,6 +780,32 @@ test "windows funky targets" {
728 );780 );
729}781}
730782
783test "windows funky prereqs" {
784 // Note we don't support unquoted escaped spaces at the very beginning of a relative path
785 // e.g. `\ SpaceAtTheBeginning.c`
786 // This typically wouldn't be seen in the wild, since depfiles usually use absolute paths
787 // and supporting it would degrade error messages for cases where it was meant to be a
788 // continuation, but the line ending is missing.
789 try depTokenizer(
790 \\cimport.o: \
791 \\ trailingbackslash\\
792 \\ C:\Users\John\ Smith\AppData\Local\zig\p\1220d14057af1a9d6dde4643293527bd5ee5099517d655251a066666a4320737ea7c\cimport.c \
793 \\ somedir\\ a.c\
794 \\ somedir/\ a.c\
795 \\ somedir\\ \ \ b.c\
796 \\ somedir\\ \\ \c.c\
797 \\
798 ,
799 \\target = {cimport.o}
800 \\prereq = {trailingbackslash\}
801 \\prereq = {C:\Users\John Smith\AppData\Local\zig\p\1220d14057af1a9d6dde4643293527bd5ee5099517d655251a066666a4320737ea7c\cimport.c}
802 \\prereq = {somedir\ a.c}
803 \\prereq = {somedir/ a.c}
804 \\prereq = {somedir\ b.c}
805 \\prereq = {somedir\ \ \c.c}
806 );
807}
808
731test "windows drive and forward slashes" {809test "windows drive and forward slashes" {
732 try depTokenizer(810 try depTokenizer(
733 \\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \811 \\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \
...@@ -915,6 +993,15 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {...@@ -915,6 +993,15 @@ fn depTokenizer(input: []const u8, expect: []const u8) !void {
915 resolve_buf.items.len = 0;993 resolve_buf.items.len = 0;
916 try buffer.appendSlice("}");994 try buffer.appendSlice("}");
917 },995 },
996 .prereq_must_resolve => {
997 try buffer.appendSlice("prereq = {");
998 try token.resolve(resolve_buf.writer());
999 for (resolve_buf.items) |b| {
1000 try buffer.append(printable_char_tab[b]);
1001 }
1002 resolve_buf.items.len = 0;
1003 try buffer.appendSlice("}");
1004 },
918 else => {1005 else => {
919 try buffer.appendSlice("ERROR: ");1006 try buffer.appendSlice("ERROR: ");
920 try token.printError(buffer.writer());1007 try token.printError(buffer.writer());