authorgravatar for phasemage@live.comPhaseMage <phasemage@live.com> 2022-01-30 11:27:52-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-01-30 21:27:52+02:00
log8a97807d6812f62db4c3088fecd04a98a84b9943
treea7658f7e1d412aa5bcede21c21b455b939074174
parent336aa3c332067ad7109a60a1276ce7a8f193ed0e
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Full response file (*.rsp) support

I hit the "quotes in an RSP file" issue when trying to compile gRPC using "zig cc". As a fun exercise, I decided to see if I could fix it myself. I'm fully open to this code being flat-out rejected. Or I can take feedback to fix it up. This modifies (and renames) _ArgIteratorWindows_ in process.zig such that it works with arbitrary strings (or the contents of an RSP file). In main.zig, this new _ArgIteratorGeneral_ is used to address the "TODO" listed in _ClangArgIterator_. This change closes #4833. **Pros:** - It has the nice attribute of handling "RSP file" arguments in the same way it handles "cmd_line" arguments. - High Performance, minimal allocations - Fixed bug in previous _ArgIteratorWindows_, where final trailing backslashes in a command line were entirely dropped - Added a test case for the above bug - Harmonized the _ArgIteratorXxxx._initWithAllocator()_ and _next()_ interface across Windows/Posix/Wasi (Moved Windows errors to _initWithAllocator()_ rather than _next()_) - Likely perf benefit on Windows by doing _utf16leToUtf8AllocZ()_ only once for the entire cmd_line **Cons:** - Breaking Change in std library on Windows: Call _ArgIterator.initWithAllocator()_ instead of _ArgIterator.init()_ - PhaseMage is new with contributions to Zig, might need a lot of hand-holding - PhaseMage is a Windows person, non-Windows stuff will need to be double-checked **Testing Done:** - Wrote a few new test cases in process.zig - zig.exe build test -Dskip-release (no new failures seen) - zig cc now builds gRPC without error

6 files changed, 337 insertions(+), 208 deletions(-)

doc/docgen.zig+5-5
......@@ -23,21 +23,21 @@ pub fn main() !void {
2323
2424 const allocator = arena.allocator();
2525
26 var args_it = process.args();
26 var args_it = try process.argsWithAllocator(allocator);
2727
2828 if (!args_it.skip()) @panic("expected self arg");
2929
30 const zig_exe = (try args_it.next(allocator)) orelse @panic("expected zig exe arg");
30 const zig_exe = args_it.next() orelse @panic("expected zig exe arg");
3131 defer allocator.free(zig_exe);
3232
33 const in_file_name = (try args_it.next(allocator)) orelse @panic("expected input arg");
33 const in_file_name = args_it.next() orelse @panic("expected input arg");
3434 defer allocator.free(in_file_name);
3535
36 const out_file_name = (try args_it.next(allocator)) orelse @panic("expected output arg");
36 const out_file_name = args_it.next() orelse @panic("expected output arg");
3737 defer allocator.free(out_file_name);
3838
3939 var do_code_tests = true;
40 if (try args_it.next(allocator)) |arg| {
40 if (args_it.next()) |arg| {
4141 if (mem.eql(u8, arg, "--skip-code-tests")) {
4242 do_code_tests = false;
4343 } else {
lib/std/process.zig+283-167
......@@ -203,6 +203,8 @@ pub const ArgIteratorPosix = struct {
203203 index: usize,
204204 count: usize,
205205
206 pub const InitError = error{};
207
206208 pub fn init() ArgIteratorPosix {
207209 return ArgIteratorPosix{
208210 .index = 0,
......@@ -299,195 +301,268 @@ pub const ArgIteratorWasi = struct {
299301 }
300302};
301303
302pub const ArgIteratorWindows = struct {
303 index: usize,
304 cmd_line: [*]const u16,
305
306 pub const NextError = error{ OutOfMemory, InvalidCmdLine };
304/// Optional parameters for `ArgIteratorGeneral`
305pub const ArgIteratorGeneralOptions = struct {
306 comments_supported: bool = false,
307};
307308
308 pub fn init() ArgIteratorWindows {
309 return initWithCmdLine(os.windows.kernel32.GetCommandLineW());
310 }
309/// A general Iterator to parse a string into a set of arguments
310pub fn ArgIteratorGeneral(comptime options: ArgIteratorGeneralOptions) type {
311 return struct {
312 allocator: Allocator,
313 index: usize = 0,
314 cmd_line: []const u8,
315
316 /// Should the cmd_line field be free'd (using the allocator) on deinit()?
317 free_cmd_line_on_deinit: bool,
318
319 /// buffer MUST be long enough to hold the cmd_line plus a null terminator.
320 /// buffer will we free'd (using the allocator) on deinit()
321 buffer: []u8,
322 start: usize = 0,
323 end: usize = 0,
324
325 pub const Self = @This();
326
327 pub const InitError = error{OutOfMemory};
328 pub const InitUtf16leError = error{ OutOfMemory, InvalidCmdLine };
329
330 /// cmd_line_utf8 MUST remain valid and constant while using this instance
331 pub fn init(allocator: Allocator, cmd_line_utf8: []const u8) InitError!Self {
332 var buffer = try allocator.alloc(u8, cmd_line_utf8.len + 1);
333 errdefer allocator.free(buffer);
334
335 return Self{
336 .allocator = allocator,
337 .cmd_line = cmd_line_utf8,
338 .free_cmd_line_on_deinit = false,
339 .buffer = buffer,
340 };
341 }
311342
312 pub fn initWithCmdLine(cmd_line: [*]const u16) ArgIteratorWindows {
313 return ArgIteratorWindows{
314 .index = 0,
315 .cmd_line = cmd_line,
316 };
317 }
343 /// cmd_line_utf8 will be free'd (with the allocator) on deinit()
344 pub fn initTakeOwnership(allocator: Allocator, cmd_line_utf8: []const u8) InitError!Self {
345 var buffer = try allocator.alloc(u8, cmd_line_utf8.len + 1);
346 errdefer allocator.free(buffer);
347
348 return Self{
349 .allocator = allocator,
350 .cmd_line = cmd_line_utf8,
351 .free_cmd_line_on_deinit = true,
352 .buffer = buffer,
353 };
354 }
318355
319 fn getPointAtIndex(self: *ArgIteratorWindows) u16 {
320 // According to
321 // https://docs.microsoft.com/en-us/windows/win32/intl/using-byte-order-marks
322 // Microsoft uses UTF16-LE. So we just read assuming it's little
323 // endian.
324 return std.mem.littleToNative(u16, self.cmd_line[self.index]);
325 }
356 /// cmd_line_utf16le MUST be encoded UTF16-LE, and is converted to UTF-8 in an internal buffer
357 pub fn initUtf16le(allocator: Allocator, cmd_line_utf16le: [*:0]const u16) InitUtf16leError!Self {
358 var utf16le_slice = mem.sliceTo(cmd_line_utf16le, 0);
359 var cmd_line = std.unicode.utf16leToUtf8Alloc(allocator, utf16le_slice) catch |err| switch (err) {
360 error.ExpectedSecondSurrogateHalf,
361 error.DanglingSurrogateHalf,
362 error.UnexpectedSecondSurrogateHalf,
363 => return error.InvalidCmdLine,
364
365 error.OutOfMemory => return error.OutOfMemory,
366 };
367 errdefer allocator.free(cmd_line);
368
369 var buffer = try allocator.alloc(u8, cmd_line.len + 1);
370 errdefer allocator.free(buffer);
371
372 return Self{
373 .allocator = allocator,
374 .cmd_line = cmd_line,
375 .free_cmd_line_on_deinit = true,
376 .buffer = buffer,
377 };
378 }
326379
327 /// You must free the returned memory when done.
328 pub fn next(self: *ArgIteratorWindows, allocator: Allocator) NextError!?[:0]u8 {
329 // march forward over whitespace
330 while (true) : (self.index += 1) {
331 const character = self.getPointAtIndex();
332 switch (character) {
333 0 => return null,
334 ' ', '\t' => continue,
335 else => break,
380 // Skips over whitespace in the cmd_line.
381 // Returns false if the terminating sentinel is reached, true otherwise.
382 // Also skips over comments (if supported).
383 fn skipWhitespace(self: *Self) bool {
384 while (true) : (self.index += 1) {
385 const character = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;
386 switch (character) {
387 0 => return false,
388 ' ', '\t', '\r', '\n' => continue,
389 '#' => {
390 if (options.comments_supported) {
391 while (true) : (self.index += 1) {
392 switch (self.cmd_line[self.index]) {
393 '\n' => break,
394 0 => return false,
395 else => continue,
396 }
397 }
398 continue;
399 } else {
400 break;
401 }
402 },
403 else => break,
404 }
336405 }
406 return true;
337407 }
338408
339 return try self.internalNext(allocator);
340 }
409 pub fn skip(self: *Self) bool {
410 if (!self.skipWhitespace()) {
411 return false;
412 }
341413
342 pub fn skip(self: *ArgIteratorWindows) bool {
343 // march forward over whitespace
344 while (true) : (self.index += 1) {
345 const character = self.getPointAtIndex();
346 switch (character) {
347 0 => return false,
348 ' ', '\t' => continue,
349 else => break,
414 var backslash_count: usize = 0;
415 var in_quote = false;
416 while (true) : (self.index += 1) {
417 const character = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;
418 switch (character) {
419 0 => return true,
420 '"' => {
421 const quote_is_real = backslash_count % 2 == 0;
422 if (quote_is_real) {
423 in_quote = !in_quote;
424 }
425 },
426 '\\' => {
427 backslash_count += 1;
428 },
429 ' ', '\t', '\r', '\n' => {
430 if (!in_quote) {
431 return true;
432 }
433 backslash_count = 0;
434 },
435 else => {
436 backslash_count = 0;
437 continue;
438 },
439 }
350440 }
351441 }
352442
353 var backslash_count: usize = 0;
354 var in_quote = false;
355 while (true) : (self.index += 1) {
356 const character = self.getPointAtIndex();
357 switch (character) {
358 0 => return true,
359 '"' => {
360 const quote_is_real = backslash_count % 2 == 0;
361 if (quote_is_real) {
362 in_quote = !in_quote;
363 }
364 },
365 '\\' => {
366 backslash_count += 1;
367 },
368 ' ', '\t' => {
369 if (!in_quote) {
370 return true;
371 }
372 backslash_count = 0;
373 },
374 else => {
375 backslash_count = 0;
376 continue;
377 },
443 /// Returns a slice of the internal buffer that contains the next argument.
444 /// Returns null when it reaches the end.
445 pub fn next(self: *Self) ?[:0]const u8 {
446 if (!self.skipWhitespace()) {
447 return null;
448 }
449
450 var backslash_count: usize = 0;
451 var in_quote = false;
452 while (true) : (self.index += 1) {
453 const character = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;
454 switch (character) {
455 0 => {
456 self.emitBackslashes(backslash_count);
457 self.buffer[self.end] = 0;
458 var token = self.buffer[self.start..self.end :0];
459 self.end += 1;
460 self.start = self.end;
461 return token;
462 },
463 '"' => {
464 const quote_is_real = backslash_count % 2 == 0;
465 self.emitBackslashes(backslash_count / 2);
466 backslash_count = 0;
467
468 if (quote_is_real) {
469 in_quote = !in_quote;
470 } else {
471 self.emitCharacter('"');
472 }
473 },
474 '\\' => {
475 backslash_count += 1;
476 },
477 ' ', '\t', '\r', '\n' => {
478 self.emitBackslashes(backslash_count);
479 backslash_count = 0;
480 if (in_quote) {
481 self.emitCharacter(character);
482 } else {
483 self.buffer[self.end] = 0;
484 var token = self.buffer[self.start..self.end :0];
485 self.end += 1;
486 self.start = self.end;
487 return token;
488 }
489 },
490 else => {
491 self.emitBackslashes(backslash_count);
492 backslash_count = 0;
493 self.emitCharacter(character);
494 },
495 }
378496 }
379497 }
380 }
381498
382 fn internalNext(self: *ArgIteratorWindows, allocator: Allocator) NextError![:0]u8 {
383 var buf = std.ArrayList(u16).init(allocator);
384 defer buf.deinit();
385
386 var backslash_count: usize = 0;
387 var in_quote = false;
388 while (true) : (self.index += 1) {
389 const character = self.getPointAtIndex();
390 switch (character) {
391 0 => {
392 return convertFromWindowsCmdLineToUTF8(allocator, buf.items);
393 },
394 '"' => {
395 const quote_is_real = backslash_count % 2 == 0;
396 try emitBackslashes(&buf, backslash_count / 2);
397 backslash_count = 0;
398
399 if (quote_is_real) {
400 in_quote = !in_quote;
401 } else {
402 try buf.append(std.mem.nativeToLittle(u16, '"'));
403 }
404 },
405 '\\' => {
406 backslash_count += 1;
407 },
408 ' ', '\t' => {
409 try emitBackslashes(&buf, backslash_count);
410 backslash_count = 0;
411 if (in_quote) {
412 try buf.append(std.mem.nativeToLittle(u16, character));
413 } else {
414 return convertFromWindowsCmdLineToUTF8(allocator, buf.items);
415 }
416 },
417 else => {
418 try emitBackslashes(&buf, backslash_count);
419 backslash_count = 0;
420 try buf.append(std.mem.nativeToLittle(u16, character));
421 },
499 fn emitBackslashes(self: *Self, emit_count: usize) void {
500 var i: usize = 0;
501 while (i < emit_count) : (i += 1) {
502 self.emitCharacter('\\');
422503 }
423504 }
424 }
425505
426 fn convertFromWindowsCmdLineToUTF8(allocator: Allocator, buf: []u16) NextError![:0]u8 {
427 return std.unicode.utf16leToUtf8AllocZ(allocator, buf) catch |err| switch (err) {
428 error.ExpectedSecondSurrogateHalf,
429 error.DanglingSurrogateHalf,
430 error.UnexpectedSecondSurrogateHalf,
431 => return error.InvalidCmdLine,
506 fn emitCharacter(self: *Self, char: u8) void {
507 self.buffer[self.end] = char;
508 self.end += 1;
509 }
432510
433 error.OutOfMemory => return error.OutOfMemory,
434 };
435 }
436 fn emitBackslashes(buf: *std.ArrayList(u16), emit_count: usize) !void {
437 var i: usize = 0;
438 while (i < emit_count) : (i += 1) {
439 try buf.append(std.mem.nativeToLittle(u16, '\\'));
511 /// Call to free the internal buffer of the iterator.
512 pub fn deinit(self: *Self) void {
513 self.allocator.free(self.buffer);
514
515 if (self.free_cmd_line_on_deinit) {
516 self.allocator.free(self.cmd_line);
517 }
440518 }
441 }
442};
519 };
520}
443521
522/// Cross-platform command line argument iterator.
444523pub const ArgIterator = struct {
445524 const InnerType = switch (builtin.os.tag) {
446 .windows => ArgIteratorWindows,
525 .windows => ArgIteratorGeneral(.{ .comments_supported = false }),
447526 .wasi => if (builtin.link_libc) ArgIteratorPosix else ArgIteratorWasi,
448527 else => ArgIteratorPosix,
449528 };
450529
451530 inner: InnerType,
452531
453 /// Initialize the args iterator.
532 /// Initialize the args iterator. Consider using initWithAllocator() instead
533 /// for cross-platform compatibility.
454534 pub fn init() ArgIterator {
455535 if (builtin.os.tag == .wasi) {
456536 @compileError("In WASI, use initWithAllocator instead.");
457537 }
538 if (builtin.os.tag == .windows) {
539 @compileError("In Windows, use initWithAllocator instead.");
540 }
458541
459542 return ArgIterator{ .inner = InnerType.init() };
460543 }
461544
462 pub const InitError = ArgIteratorWasi.InitError;
545 pub const InitError = switch (builtin.os.tag) {
546 .windows => InnerType.InitUtf16leError,
547 else => InnerType.InitError,
548 };
463549
464550 /// You must deinitialize iterator's internal buffers by calling `deinit` when done.
465551 pub fn initWithAllocator(allocator: mem.Allocator) InitError!ArgIterator {
466552 if (builtin.os.tag == .wasi and !builtin.link_libc) {
467553 return ArgIterator{ .inner = try InnerType.init(allocator) };
468554 }
469
470 return ArgIterator{ .inner = InnerType.init() };
471 }
472
473 pub const NextError = ArgIteratorWindows.NextError;
474
475 /// You must free the returned memory when done.
476 pub fn next(self: *ArgIterator, allocator: Allocator) NextError!?[:0]u8 {
477555 if (builtin.os.tag == .windows) {
478 return self.inner.next(allocator);
479 } else {
480 return try allocator.dupeZ(u8, self.inner.next() orelse return null);
556 const cmd_line_w = os.windows.kernel32.GetCommandLineW();
557 return ArgIterator{ .inner = try InnerType.initUtf16le(allocator, cmd_line_w) };
481558 }
482 }
483559
484 /// If you only are targeting posix you can call this and not need an allocator.
485 pub fn nextPosix(self: *ArgIterator) ?[:0]const u8 {
486 return self.inner.next();
560 return ArgIterator{ .inner = InnerType.init() };
487561 }
488562
489 /// If you only are targeting WASI, you can call this and not need an allocator.
490 pub fn nextWasi(self: *ArgIterator) ?[:0]const u8 {
563 /// Get the next argument. Returns 'null' if we are at the end.
564 /// Returned slice is pointing to the iterator's internal buffer.
565 pub fn next(self: *ArgIterator) ?([:0]const u8) {
491566 return self.inner.next();
492567 }
493568
......@@ -500,13 +575,18 @@ pub const ArgIterator = struct {
500575 /// Call this to free the iterator's internal buffer if the iterator
501576 /// was created with `initWithAllocator` function.
502577 pub fn deinit(self: *ArgIterator) void {
503 // Unless we're targeting WASI, this is a no-op.
578 // Unless we're targeting WASI or Windows, this is a no-op.
504579 if (builtin.os.tag == .wasi and !builtin.link_libc) {
505580 self.inner.deinit();
506581 }
582
583 if (builtin.os.tag == .windows) {
584 self.inner.deinit();
585 }
507586 }
508587};
509588
589/// Use argsWithAllocator() for cross-platform code
510590pub fn args() ArgIterator {
511591 return ArgIterator.init();
512592}
......@@ -518,12 +598,10 @@ pub fn argsWithAllocator(allocator: mem.Allocator) ArgIterator.InitError!ArgIter
518598
519599test "args iterator" {
520600 var ga = std.testing.allocator;
521 var it = if (builtin.os.tag == .wasi) try argsWithAllocator(ga) else args();
522 defer it.deinit(); // no-op unless WASI
523
524 const prog_name = (try it.next(ga)) orelse unreachable;
525 defer ga.free(prog_name);
601 var it = try argsWithAllocator(ga);
602 defer it.deinit(); // no-op unless WASI or Windows
526603
604 const prog_name = it.next() orelse unreachable;
527605 const expected_suffix = switch (builtin.os.tag) {
528606 .wasi => "test.wasm",
529607 .windows => "test.exe",
......@@ -533,14 +611,14 @@ test "args iterator" {
533611
534612 try testing.expect(mem.eql(u8, expected_suffix, given_suffix));
535613 try testing.expect(it.skip()); // Skip over zig_exe_path, passed to the test runner
536 try testing.expect((try it.next(ga)) == null);
614 try testing.expect(it.next() == null);
537615 try testing.expect(!it.skip());
538616}
539617
540618/// Caller must call argsFree on result.
541619pub fn argsAlloc(allocator: mem.Allocator) ![][:0]u8 {
542620 // TODO refactor to only make 1 allocation.
543 var it = if (builtin.os.tag == .wasi) try argsWithAllocator(allocator) else args();
621 var it = try argsWithAllocator(allocator);
544622 defer it.deinit();
545623
546624 var contents = std.ArrayList(u8).init(allocator);
......@@ -549,8 +627,7 @@ pub fn argsAlloc(allocator: mem.Allocator) ![][:0]u8 {
549627 var slice_list = std.ArrayList(usize).init(allocator);
550628 defer slice_list.deinit();
551629
552 while (try it.next(allocator)) |arg| {
553 defer allocator.free(arg);
630 while (it.next()) |arg| {
554631 try contents.appendSlice(arg[0 .. arg.len + 1]);
555632 try slice_list.append(arg.len);
556633 }
......@@ -586,16 +663,17 @@ pub fn argsFree(allocator: mem.Allocator, args_alloc: []const [:0]u8) void {
586663 return allocator.free(aligned_allocated_buf);
587664}
588665
589test "windows arg parsing" {
590 const utf16Literal = std.unicode.utf8ToUtf16LeStringLiteral;
591 try testWindowsCmdLine(utf16Literal("a b\tc d"), &[_][]const u8{ "a", "b", "c", "d" });
592 try testWindowsCmdLine(utf16Literal("\"abc\" d e"), &[_][]const u8{ "abc", "d", "e" });
593 try testWindowsCmdLine(utf16Literal("a\\\\\\b d\"e f\"g h"), &[_][]const u8{ "a\\\\\\b", "de fg", "h" });
594 try testWindowsCmdLine(utf16Literal("a\\\\\\\"b c d"), &[_][]const u8{ "a\\\"b", "c", "d" });
595 try testWindowsCmdLine(utf16Literal("a\\\\\\\\\"b c\" d e"), &[_][]const u8{ "a\\\\b c", "d", "e" });
596 try testWindowsCmdLine(utf16Literal("a b\tc \"d f"), &[_][]const u8{ "a", "b", "c", "d f" });
597
598 try testWindowsCmdLine(utf16Literal("\".\\..\\zig-cache\\build\" \"bin\\zig.exe\" \".\\..\" \".\\..\\zig-cache\" \"--help\""), &[_][]const u8{
666test "general arg parsing" {
667 try testGeneralCmdLine("a b\tc d", &[_][]const u8{ "a", "b", "c", "d" });
668 try testGeneralCmdLine("\"abc\" d e", &[_][]const u8{ "abc", "d", "e" });
669 try testGeneralCmdLine("a\\\\\\b d\"e f\"g h", &[_][]const u8{ "a\\\\\\b", "de fg", "h" });
670 try testGeneralCmdLine("a\\\\\\\"b c d", &[_][]const u8{ "a\\\"b", "c", "d" });
671 try testGeneralCmdLine("a\\\\\\\\\"b c\" d e", &[_][]const u8{ "a\\\\b c", "d", "e" });
672 try testGeneralCmdLine("a b\tc \"d f", &[_][]const u8{ "a", "b", "c", "d f" });
673 try testGeneralCmdLine("j k l\\", &[_][]const u8{ "j", "k", "l\\" });
674 try testGeneralCmdLine("\"\" x y z\\\\", &[_][]const u8{ "", "x", "y", "z\\\\" });
675
676 try testGeneralCmdLine("\".\\..\\zig-cache\\build\" \"bin\\zig.exe\" \".\\..\" \".\\..\\zig-cache\" \"--help\"", &[_][]const u8{
599677 ".\\..\\zig-cache\\build",
600678 "bin\\zig.exe",
601679 ".\\..",
......@@ -604,14 +682,52 @@ test "windows arg parsing" {
604682 });
605683}
606684
607fn testWindowsCmdLine(input_cmd_line: [*]const u16, expected_args: []const []const u8) !void {
608 var it = ArgIteratorWindows.initWithCmdLine(input_cmd_line);
685fn testGeneralCmdLine(input_cmd_line: []const u8, expected_args: []const []const u8) !void {
686 var it = try ArgIteratorGeneral(.{ .comments_supported = false })
687 .init(std.testing.allocator, input_cmd_line);
688 defer it.deinit();
689 for (expected_args) |expected_arg| {
690 const arg = it.next().?;
691 try testing.expectEqualStrings(expected_arg, arg);
692 }
693 try testing.expect(it.next() == null);
694}
695
696test "response file arg parsing" {
697 try testResponseFileCmdLine(
698 \\a b
699 \\c d\
700 , &[_][]const u8{ "a", "b", "c", "d\\" });
701 try testResponseFileCmdLine("a b c d\\", &[_][]const u8{ "a", "b", "c", "d\\" });
702
703 try testResponseFileCmdLine(
704 \\j
705 \\ k l # this is a comment \\ \\\ \\\\ "none" "\\" "\\\"
706 \\ "m" #another comment
707 \\
708 , &[_][]const u8{ "j", "k", "l", "m" });
709
710 try testResponseFileCmdLine(
711 \\ "" q ""
712 \\ "r s # t" "u\" v" #another comment
713 \\
714 , &[_][]const u8{ "", "q", "", "r s # t", "u\" v" });
715
716 try testResponseFileCmdLine(
717 \\ -l"advapi32" a# b#c d#
718 \\e\\\
719 , &[_][]const u8{ "-ladvapi32", "a#", "b#c", "d#", "e\\\\\\" });
720}
721
722fn testResponseFileCmdLine(input_cmd_line: []const u8, expected_args: []const []const u8) !void {
723 var it = try ArgIteratorGeneral(.{ .comments_supported = true })
724 .init(std.testing.allocator, input_cmd_line);
725 defer it.deinit();
609726 for (expected_args) |expected_arg| {
610 const arg = (it.next(std.testing.allocator) catch unreachable).?;
611 defer std.testing.allocator.free(arg);
727 const arg = it.next().?;
612728 try testing.expectEqualStrings(expected_arg, arg);
613729 }
614 try testing.expect((try it.next(std.testing.allocator)) == null);
730 try testing.expect(it.next() == null);
615731}
616732
617733pub const UserInfo = struct {
lib/std/unicode.zig+2-2
......@@ -568,8 +568,8 @@ pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8
568568
569569/// Caller must free returned memory.
570570pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0]u8 {
571 // optimistically guess that it will all be ascii.
572 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);
571 // optimistically guess that it will all be ascii (and allocate space for the null terminator)
572 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);
573573 errdefer result.deinit();
574574 var out_index: usize = 0;
575575 var it = Utf16LeIterator.init(utf16le);
src/main.zig+32-24
......@@ -4148,7 +4148,8 @@ pub const ClangArgIterator = struct {
41484148 argv: []const []const u8,
41494149 next_index: usize,
41504150 root_args: ?*Args,
4151 allocator: Allocator,
4151 arg_iterator_response_file: ArgIteratorResponseFile,
4152 arena: Allocator,
41524153
41534154 pub const ZigEquivalent = enum {
41544155 target,
......@@ -4210,7 +4211,7 @@ pub const ClangArgIterator = struct {
42104211 argv: []const []const u8,
42114212 };
42124213
4213 fn init(allocator: Allocator, argv: []const []const u8) ClangArgIterator {
4214 fn init(arena: Allocator, argv: []const []const u8) ClangArgIterator {
42144215 return .{
42154216 .next_index = 2, // `zig cc foo` this points to `foo`
42164217 .has_next = argv.len > 2,
......@@ -4220,10 +4221,22 @@ pub const ClangArgIterator = struct {
42204221 .other_args = undefined,
42214222 .argv = argv,
42224223 .root_args = null,
4223 .allocator = allocator,
4224 .arg_iterator_response_file = undefined,
4225 .arena = arena,
42244226 };
42254227 }
42264228
4229 const ArgIteratorResponseFile = process.ArgIteratorGeneral(.{ .comments_supported = true });
4230
4231 /// Initialize the arguments from a Response File. "*.rsp"
4232 fn initArgIteratorResponseFile(allocator: Allocator, resp_file_path: []const u8) !ArgIteratorResponseFile {
4233 const max_bytes = 10 * 1024 * 1024; // 10 MiB of command line arguments is a reasonable limit
4234 var cmd_line = try fs.cwd().readFileAlloc(allocator, resp_file_path, max_bytes);
4235 errdefer allocator.free(cmd_line);
4236
4237 return ArgIteratorResponseFile.initTakeOwnership(allocator, cmd_line);
4238 }
4239
42274240 fn next(self: *ClangArgIterator) !void {
42284241 assert(self.has_next);
42294242 assert(self.next_index < self.argv.len);
......@@ -4239,31 +4252,25 @@ pub const ClangArgIterator = struct {
42394252
42404253 // This is a "compiler response file". We must parse the file and treat its
42414254 // contents as command line parameters.
4242 const allocator = self.allocator;
4243 const max_bytes = 10 * 1024 * 1024; // 10 MiB of command line arguments is a reasonable limit
4255 const arena = self.arena;
42444256 const resp_file_path = arg[1..];
4245 const resp_contents = fs.cwd().readFileAlloc(allocator, resp_file_path, max_bytes) catch |err| {
4257
4258 self.arg_iterator_response_file =
4259 initArgIteratorResponseFile(arena, resp_file_path) catch |err| {
42464260 fatal("unable to read response file '{s}': {s}", .{ resp_file_path, @errorName(err) });
42474261 };
4248 defer allocator.free(resp_contents);
4249 // TODO is there a specification for this file format? Let's find it and make this parsing more robust
4250 // at the very least I'm guessing this needs to handle quotes and `#` comments.
4251 var it = mem.tokenize(u8, resp_contents, " \t\r\n");
4252 var resp_arg_list = std.ArrayList([]const u8).init(allocator);
4262 // NOTE: The ArgIteratorResponseFile returns tokens from next() that are slices of an
4263 // internal buffer. This internal buffer is arena allocated, so it is not cleaned up here.
4264
4265 var resp_arg_list = std.ArrayList([]const u8).init(arena);
42534266 defer resp_arg_list.deinit();
42544267 {
4255 errdefer {
4256 for (resp_arg_list.items) |item| {
4257 allocator.free(mem.span(item));
4258 }
4268 while (self.arg_iterator_response_file.next()) |token| {
4269 try resp_arg_list.append(token);
42594270 }
4260 while (it.next()) |token| {
4261 const dupe_token = try allocator.dupeZ(u8, token);
4262 errdefer allocator.free(dupe_token);
4263 try resp_arg_list.append(dupe_token);
4264 }
4265 const args = try allocator.create(Args);
4266 errdefer allocator.destroy(args);
4271
4272 const args = try arena.create(Args);
4273 errdefer arena.destroy(args);
42674274 args.* = .{
42684275 .next_index = self.next_index,
42694276 .argv = self.argv,
......@@ -4284,6 +4291,7 @@ pub const ClangArgIterator = struct {
42844291 arg = mem.span(self.argv[self.next_index]);
42854292 self.incrementArgIndex();
42864293 }
4294
42874295 if (mem.eql(u8, arg, "-") or !mem.startsWith(u8, arg, "-")) {
42884296 self.zig_equivalent = .positional;
42894297 self.only_arg = arg;
......@@ -4383,13 +4391,13 @@ pub const ClangArgIterator = struct {
43834391 }
43844392
43854393 fn resolveRespFileArgs(self: *ClangArgIterator) void {
4386 const allocator = self.allocator;
4394 const arena = self.arena;
43874395 if (self.next_index >= self.argv.len) {
43884396 if (self.root_args) |root_args| {
43894397 self.next_index = root_args.next_index;
43904398 self.argv = root_args.argv;
43914399
4392 allocator.destroy(root_args);
4400 arena.destroy(root_args);
43934401 self.root_args = null;
43944402 }
43954403 if (self.next_index >= self.argv.len) {
test/cli.zig+4-5
......@@ -11,18 +11,17 @@ pub fn main() !void {
1111 var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
1212 defer arena.deinit();
1313
14 var arg_it = process.args();
14 a = arena.allocator();
15 var arg_it = try process.argsWithAllocator(a);
1516
1617 // skip my own exe name
1718 _ = arg_it.skip();
1819
19 a = arena.allocator();
20
21 const zig_exe_rel = (try arg_it.next(a)) orelse {
20 const zig_exe_rel = arg_it.next() orelse {
2221 std.debug.print("Expected first argument to be path to zig compiler\n", .{});
2322 return error.InvalidArgs;
2423 };
25 const cache_root = (try arg_it.next(a)) orelse {
24 const cache_root = arg_it.next() orelse {
2625 std.debug.print("Expected second argument to be cache root directory path\n", .{});
2726 return error.InvalidArgs;
2827 };
test/compare_output.zig+11-5
......@@ -291,7 +291,9 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
291291 \\ stdout.print("before\n", .{}) catch unreachable;
292292 \\ defer stdout.print("defer1\n", .{}) catch unreachable;
293293 \\ defer stdout.print("defer2\n", .{}) catch unreachable;
294 \\ var args_it = @import("std").process.args();
294 \\ var arena = @import("std").heap.ArenaAllocator.init(@import("std").testing.allocator);
295 \\ defer arena.deinit();
296 \\ var args_it = @import("std").process.argsWithAllocator(arena.allocator()) catch unreachable;
295297 \\ if (args_it.skip() and !args_it.skip()) return;
296298 \\ defer stdout.print("defer3\n", .{}) catch unreachable;
297299 \\ stdout.print("after\n", .{}) catch unreachable;
......@@ -358,11 +360,13 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
358360 \\const allocator = std.testing.allocator;
359361 \\
360362 \\pub fn main() !void {
361 \\ var args_it = std.process.args();
363 \\ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
364 \\ defer arena.deinit();
365 \\ var args_it = try std.process.argsWithAllocator(arena.allocator());
362366 \\ const stdout = io.getStdOut().writer();
363367 \\ var index: usize = 0;
364368 \\ _ = args_it.skip();
365 \\ while (try args_it.next(allocator)) |arg| : (index += 1) {
369 \\ while (args_it.next()) |arg| : (index += 1) {
366370 \\ try stdout.print("{}: {s}\n", .{index, arg});
367371 \\ }
368372 \\}
......@@ -396,11 +400,13 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
396400 \\const allocator = std.testing.allocator;
397401 \\
398402 \\pub fn main() !void {
399 \\ var args_it = std.process.args();
403 \\ var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
404 \\ defer arena.deinit();
405 \\ var args_it = try std.process.argsWithAllocator(arena.allocator());
400406 \\ const stdout = io.getStdOut().writer();
401407 \\ var index: usize = 0;
402408 \\ _ = args_it.skip();
403 \\ while (try args_it.next(allocator)) |arg| : (index += 1) {
409 \\ while (args_it.next()) |arg| : (index += 1) {
404410 \\ try stdout.print("{}: {s}\n", .{index, arg});
405411 \\ }
406412 \\}