Commit 8a97807d6812f62db4c3088fecd04a98a84b9943 | zig

author	PhaseMage <phasemage@live.com> 2022-01-30 11:27:52-08:00
committer	GitHub <noreply@github.com> 2022-01-30 21:27:52+02:00
log	8a97807d6812f62db4c3088fecd04a98a84b9943
tree	a7658f7e1d412aa5bcede21c21b455b939074174
parent	336aa3c332067ad7109a60a1276ce7a8f193ed0e
signature	Signed by PGP key 4AEE18F83AFDEB23

Full response file (*.rsp) support

I hit the "quotes in an RSP file" issue when trying to compile gRPC using
"zig cc". As a fun exercise, I decided to see if I could fix it myself.
I'm fully open to this code being flat-out rejected. Or I can take feedback
to fix it up.

This modifies (and renames) _ArgIteratorWindows_ in process.zig such that
it works with arbitrary strings (or the contents of an RSP file).

In main.zig, this new _ArgIteratorGeneral_ is used to address the "TODO"
listed in _ClangArgIterator_.

This change closes #4833.

**Pros:**

- It has the nice attribute of handling "RSP file" arguments in the same way it
  handles "cmd_line" arguments.
- High Performance, minimal allocations
- Fixed bug in previous _ArgIteratorWindows_, where final trailing backslashes
  in a command line were entirely dropped
- Added a test case for the above bug
- Harmonized the _ArgIteratorXxxx._initWithAllocator()_ and _next()_ interface
  across Windows/Posix/Wasi (Moved Windows errors to _initWithAllocator()_
  rather than _next()_)
- Likely perf benefit on Windows by doing _utf16leToUtf8AllocZ()_ only once
  for the entire cmd_line

**Cons:**

- Breaking Change in std library on Windows: Call
  _ArgIterator.initWithAllocator()_ instead of _ArgIterator.init()_
- PhaseMage is new with contributions to Zig, might need a lot of hand-holding
- PhaseMage is a Windows person, non-Windows stuff will need to be double-checked

**Testing Done:**

- Wrote a few new test cases in process.zig
- zig.exe build test -Dskip-release (no new failures seen)
- zig cc now builds gRPC without error

6 files changed, 337 insertions(+), 208 deletions(-)

doc
- docgen.zig
lib
- std
  - process.zig
  - unicode.zig
src
- main.zig
test
- cli.zig
- compare_output.zig

doc/docgen.zig+5-5

...	@@ -23,21 +23,21 @@ pub fn main() !void {	...	@@ -23,21 +23,21 @@ pub fn main() !void {
23		23
24	const allocator = arena.allocator();	24	const allocator = arena.allocator();
25		25
26	var args_it = process.args();	26	var args_it = try process.argsWithAllocator(allocator);
27		27
28	if (!args_it.skip()) @panic("expected self arg");	28	if (!args_it.skip()) @panic("expected self arg");
29		29
30	const zig_exe = (try args_it.next(allocator)) orelse @panic("expected zig exe arg");	30	const zig_exe = args_it.next() orelse @panic("expected zig exe arg");
31	defer allocator.free(zig_exe);	31	defer allocator.free(zig_exe);
32		32
33	const in_file_name = (try args_it.next(allocator)) orelse @panic("expected input arg");	33	const in_file_name = args_it.next() orelse @panic("expected input arg");
34	defer allocator.free(in_file_name);	34	defer allocator.free(in_file_name);
35		35
36	const out_file_name = (try args_it.next(allocator)) orelse @panic("expected output arg");	36	const out_file_name = args_it.next() orelse @panic("expected output arg");
37	defer allocator.free(out_file_name);	37	defer allocator.free(out_file_name);
38		38
39	var do_code_tests = true;	39	var do_code_tests = true;
40	if (try args_it.next(allocator)) \|arg\| {	40	if (args_it.next()) \|arg\| {
41	if (mem.eql(u8, arg, "--skip-code-tests")) {	41	if (mem.eql(u8, arg, "--skip-code-tests")) {
42	do_code_tests = false;	42	do_code_tests = false;
43	} else {	43	} else {

lib/std/process.zig+283-167

...	@@ -203,6 +203,8 @@ pub const ArgIteratorPosix = struct {	...	@@ -203,6 +203,8 @@ pub const ArgIteratorPosix = struct {
203	index: usize,	203	index: usize,
204	count: usize,	204	count: usize,
205		205
		206	pub const InitError = error{};
		207
206	pub fn init() ArgIteratorPosix {	208	pub fn init() ArgIteratorPosix {
207	return ArgIteratorPosix{	209	return ArgIteratorPosix{
208	.index = 0,	210	.index = 0,
...	@@ -299,195 +301,268 @@ pub const ArgIteratorWasi = struct {	...	@@ -299,195 +301,268 @@ pub const ArgIteratorWasi = struct {
299	}	301	}
300	};	302	};
301		303
302	pub const ArgIteratorWindows = struct {	304	/// Optional parameters for `ArgIteratorGeneral`
303	index: usize,	305	pub const ArgIteratorGeneralOptions = struct {
304	cmd_line: [*]const u16,	306	comments_supported: bool = false,
305		307	};
306	pub const NextError = error{ OutOfMemory, InvalidCmdLine };
307		308
308	pub fn init() ArgIteratorWindows {	309	/// A general Iterator to parse a string into a set of arguments
309	return initWithCmdLine(os.windows.kernel32.GetCommandLineW());	310	pub fn ArgIteratorGeneral(comptime options: ArgIteratorGeneralOptions) type {
310	}	311	return struct {
		312	allocator: Allocator,
		313	index: usize = 0,
		314	cmd_line: []const u8,
		315
		316	/// Should the cmd_line field be free'd (using the allocator) on deinit()?
		317	free_cmd_line_on_deinit: bool,
		318
		319	/// buffer MUST be long enough to hold the cmd_line plus a null terminator.
		320	/// buffer will we free'd (using the allocator) on deinit()
		321	buffer: []u8,
		322	start: usize = 0,
		323	end: usize = 0,
		324
		325	pub const Self = @This();
		326
		327	pub const InitError = error{OutOfMemory};
		328	pub const InitUtf16leError = error{ OutOfMemory, InvalidCmdLine };
		329
		330	/// cmd_line_utf8 MUST remain valid and constant while using this instance
		331	pub fn init(allocator: Allocator, cmd_line_utf8: []const u8) InitError!Self {
		332	var buffer = try allocator.alloc(u8, cmd_line_utf8.len + 1);
		333	errdefer allocator.free(buffer);
		334
		335	return Self{
		336	.allocator = allocator,
		337	.cmd_line = cmd_line_utf8,
		338	.free_cmd_line_on_deinit = false,
		339	.buffer = buffer,
		340	};
		341	}
311		342
312	pub fn initWithCmdLine(cmd_line: [*]const u16) ArgIteratorWindows {	343	/// cmd_line_utf8 will be free'd (with the allocator) on deinit()
313	return ArgIteratorWindows{	344	pub fn initTakeOwnership(allocator: Allocator, cmd_line_utf8: []const u8) InitError!Self {
314	.index = 0,	345	var buffer = try allocator.alloc(u8, cmd_line_utf8.len + 1);
315	.cmd_line = cmd_line,	346	errdefer allocator.free(buffer);
316	};	347
317	}	348	return Self{
		349	.allocator = allocator,
		350	.cmd_line = cmd_line_utf8,
		351	.free_cmd_line_on_deinit = true,
		352	.buffer = buffer,
		353	};
		354	}
318		355
319	fn getPointAtIndex(self: *ArgIteratorWindows) u16 {	356	/// cmd_line_utf16le MUST be encoded UTF16-LE, and is converted to UTF-8 in an internal buffer
320	// According to	357	pub fn initUtf16le(allocator: Allocator, cmd_line_utf16le: [*:0]const u16) InitUtf16leError!Self {
321	// https://docs.microsoft.com/en-us/windows/win32/intl/using-byte-order-marks	358	var utf16le_slice = mem.sliceTo(cmd_line_utf16le, 0);
322	// Microsoft uses UTF16-LE. So we just read assuming it's little	359	var cmd_line = std.unicode.utf16leToUtf8Alloc(allocator, utf16le_slice) catch \|err\| switch (err) {
323	// endian.	360	error.ExpectedSecondSurrogateHalf,
324	return std.mem.littleToNative(u16, self.cmd_line[self.index]);	361	error.DanglingSurrogateHalf,
325	}	362	error.UnexpectedSecondSurrogateHalf,
		363	=> return error.InvalidCmdLine,
		364
		365	error.OutOfMemory => return error.OutOfMemory,
		366	};
		367	errdefer allocator.free(cmd_line);
		368
		369	var buffer = try allocator.alloc(u8, cmd_line.len + 1);
		370	errdefer allocator.free(buffer);
		371
		372	return Self{
		373	.allocator = allocator,
		374	.cmd_line = cmd_line,
		375	.free_cmd_line_on_deinit = true,
		376	.buffer = buffer,
		377	};
		378	}
326		379
327	/// You must free the returned memory when done.	380	// Skips over whitespace in the cmd_line.
328	pub fn next(self: *ArgIteratorWindows, allocator: Allocator) NextError!?[:0]u8 {	381	// Returns false if the terminating sentinel is reached, true otherwise.
329	// march forward over whitespace	382	// Also skips over comments (if supported).
330	while (true) : (self.index += 1) {	383	fn skipWhitespace(self: *Self) bool {
331	const character = self.getPointAtIndex();	384	while (true) : (self.index += 1) {
332	switch (character) {	385	const character = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;
333	0 => return null,	386	switch (character) {
334	' ', '\t' => continue,	387	0 => return false,
335	else => break,	388	' ', '\t', '\r', '\n' => continue,
		389	'#' => {
		390	if (options.comments_supported) {
		391	while (true) : (self.index += 1) {
		392	switch (self.cmd_line[self.index]) {
		393	'\n' => break,
		394	0 => return false,
		395	else => continue,
		396	}
		397	}
		398	continue;
		399	} else {
		400	break;
		401	}
		402	},
		403	else => break,
		404	}
336	}	405	}
		406	return true;
337	}	407	}
338		408
339	return try self.internalNext(allocator);	409	pub fn skip(self: *Self) bool {
340	}	410	if (!self.skipWhitespace()) {
		411	return false;
		412	}
341		413
342	pub fn skip(self: *ArgIteratorWindows) bool {	414	var backslash_count: usize = 0;
343	// march forward over whitespace	415	var in_quote = false;
344	while (true) : (self.index += 1) {	416	while (true) : (self.index += 1) {
345	const character = self.getPointAtIndex();	417	const character = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;
346	switch (character) {	418	switch (character) {
347	0 => return false,	419	0 => return true,
348	' ', '\t' => continue,	420	'"' => {
349	else => break,	421	const quote_is_real = backslash_count % 2 == 0;
		422	if (quote_is_real) {
		423	in_quote = !in_quote;
		424	}
		425	},
		426	'\\' => {
		427	backslash_count += 1;
		428	},
		429	' ', '\t', '\r', '\n' => {
		430	if (!in_quote) {
		431	return true;
		432	}
		433	backslash_count = 0;
		434	},
		435	else => {
		436	backslash_count = 0;
		437	continue;
		438	},
		439	}
350	}	440	}
351	}	441	}
352		442
353	var backslash_count: usize = 0;	443	/// Returns a slice of the internal buffer that contains the next argument.
354	var in_quote = false;	444	/// Returns null when it reaches the end.
355	while (true) : (self.index += 1) {	445	pub fn next(self: *Self) ?[:0]const u8 {
356	const character = self.getPointAtIndex();	446	if (!self.skipWhitespace()) {
357	switch (character) {	447	return null;
358	0 => return true,	448	}
359	'"' => {	449
360	const quote_is_real = backslash_count % 2 == 0;	450	var backslash_count: usize = 0;
361	if (quote_is_real) {	451	var in_quote = false;
362	in_quote = !in_quote;	452	while (true) : (self.index += 1) {
363	}	453	const character = if (self.index != self.cmd_line.len) self.cmd_line[self.index] else 0;
364	},	454	switch (character) {
365	'\\' => {	455	0 => {
366	backslash_count += 1;	456	self.emitBackslashes(backslash_count);
367	},	457	self.buffer[self.end] = 0;
368	' ', '\t' => {	458	var token = self.buffer[self.start..self.end :0];
369	if (!in_quote) {	459	self.end += 1;
370	return true;	460	self.start = self.end;
371	}	461	return token;
372	backslash_count = 0;	462	},
373	},	463	'"' => {
374	else => {	464	const quote_is_real = backslash_count % 2 == 0;
375	backslash_count = 0;	465	self.emitBackslashes(backslash_count / 2);
376	continue;	466	backslash_count = 0;
377	},	467
		468	if (quote_is_real) {
		469	in_quote = !in_quote;
		470	} else {
		471	self.emitCharacter('"');
		472	}
		473	},
		474	'\\' => {
		475	backslash_count += 1;
		476	},
		477	' ', '\t', '\r', '\n' => {
		478	self.emitBackslashes(backslash_count);
		479	backslash_count = 0;
		480	if (in_quote) {
		481	self.emitCharacter(character);
		482	} else {
		483	self.buffer[self.end] = 0;
		484	var token = self.buffer[self.start..self.end :0];
		485	self.end += 1;
		486	self.start = self.end;
		487	return token;
		488	}
		489	},
		490	else => {
		491	self.emitBackslashes(backslash_count);
		492	backslash_count = 0;
		493	self.emitCharacter(character);
		494	},
		495	}
378	}	496	}
379	}	497	}
380	}
381		498
382	fn internalNext(self: *ArgIteratorWindows, allocator: Allocator) NextError![:0]u8 {	499	fn emitBackslashes(self: *Self, emit_count: usize) void {
383	var buf = std.ArrayList(u16).init(allocator);	500	var i: usize = 0;
384	defer buf.deinit();	501	while (i < emit_count) : (i += 1) {
385		502	self.emitCharacter('\\');
386	var backslash_count: usize = 0;
387	var in_quote = false;
388	while (true) : (self.index += 1) {
389	const character = self.getPointAtIndex();
390	switch (character) {
391	0 => {
392	return convertFromWindowsCmdLineToUTF8(allocator, buf.items);
393	},
394	'"' => {
395	const quote_is_real = backslash_count % 2 == 0;
396	try emitBackslashes(&buf, backslash_count / 2);
397	backslash_count = 0;
398
399	if (quote_is_real) {
400	in_quote = !in_quote;
401	} else {
402	try buf.append(std.mem.nativeToLittle(u16, '"'));
403	}
404	},
405	'\\' => {
406	backslash_count += 1;
407	},
408	' ', '\t' => {
409	try emitBackslashes(&buf, backslash_count);
410	backslash_count = 0;
411	if (in_quote) {
412	try buf.append(std.mem.nativeToLittle(u16, character));
413	} else {
414	return convertFromWindowsCmdLineToUTF8(allocator, buf.items);
415	}
416	},
417	else => {
418	try emitBackslashes(&buf, backslash_count);
419	backslash_count = 0;
420	try buf.append(std.mem.nativeToLittle(u16, character));
421	},
422	}	503	}
423	}	504	}
424	}
425		505
426	fn convertFromWindowsCmdLineToUTF8(allocator: Allocator, buf: []u16) NextError![:0]u8 {	506	fn emitCharacter(self: *Self, char: u8) void {
427	return std.unicode.utf16leToUtf8AllocZ(allocator, buf) catch \|err\| switch (err) {	507	self.buffer[self.end] = char;
428	error.ExpectedSecondSurrogateHalf,	508	self.end += 1;
429	error.DanglingSurrogateHalf,	509	}
430	error.UnexpectedSecondSurrogateHalf,
431	=> return error.InvalidCmdLine,
432		510
433	error.OutOfMemory => return error.OutOfMemory,	511	/// Call to free the internal buffer of the iterator.
434	};	512	pub fn deinit(self: *Self) void {
435	}	513	self.allocator.free(self.buffer);
436	fn emitBackslashes(buf: *std.ArrayList(u16), emit_count: usize) !void {	514
437	var i: usize = 0;	515	if (self.free_cmd_line_on_deinit) {
438	while (i < emit_count) : (i += 1) {	516	self.allocator.free(self.cmd_line);
439	try buf.append(std.mem.nativeToLittle(u16, '\\'));	517	}
440	}	518	}
441	}	519	};
442	};	520	}
443		521
		522	/// Cross-platform command line argument iterator.
444	pub const ArgIterator = struct {	523	pub const ArgIterator = struct {
445	const InnerType = switch (builtin.os.tag) {	524	const InnerType = switch (builtin.os.tag) {
446	.windows => ArgIteratorWindows,	525	.windows => ArgIteratorGeneral(.{ .comments_supported = false }),
447	.wasi => if (builtin.link_libc) ArgIteratorPosix else ArgIteratorWasi,	526	.wasi => if (builtin.link_libc) ArgIteratorPosix else ArgIteratorWasi,
448	else => ArgIteratorPosix,	527	else => ArgIteratorPosix,
449	};	528	};
450		529
451	inner: InnerType,	530	inner: InnerType,
452		531
453	/// Initialize the args iterator.	532	/// Initialize the args iterator. Consider using initWithAllocator() instead
		533	/// for cross-platform compatibility.
454	pub fn init() ArgIterator {	534	pub fn init() ArgIterator {
455	if (builtin.os.tag == .wasi) {	535	if (builtin.os.tag == .wasi) {
456	@compileError("In WASI, use initWithAllocator instead.");	536	@compileError("In WASI, use initWithAllocator instead.");
457	}	537	}
		538	if (builtin.os.tag == .windows) {
		539	@compileError("In Windows, use initWithAllocator instead.");
		540	}
458		541
459	return ArgIterator{ .inner = InnerType.init() };	542	return ArgIterator{ .inner = InnerType.init() };
460	}	543	}
461		544
462	pub const InitError = ArgIteratorWasi.InitError;	545	pub const InitError = switch (builtin.os.tag) {
		546	.windows => InnerType.InitUtf16leError,
		547	else => InnerType.InitError,
		548	};
463		549
464	/// You must deinitialize iterator's internal buffers by calling `deinit` when done.	550	/// You must deinitialize iterator's internal buffers by calling `deinit` when done.
465	pub fn initWithAllocator(allocator: mem.Allocator) InitError!ArgIterator {	551	pub fn initWithAllocator(allocator: mem.Allocator) InitError!ArgIterator {
466	if (builtin.os.tag == .wasi and !builtin.link_libc) {	552	if (builtin.os.tag == .wasi and !builtin.link_libc) {
467	return ArgIterator{ .inner = try InnerType.init(allocator) };	553	return ArgIterator{ .inner = try InnerType.init(allocator) };
468	}	554	}
469
470	return ArgIterator{ .inner = InnerType.init() };
471	}
472
473	pub const NextError = ArgIteratorWindows.NextError;
474
475	/// You must free the returned memory when done.
476	pub fn next(self: *ArgIterator, allocator: Allocator) NextError!?[:0]u8 {
477	if (builtin.os.tag == .windows) {	555	if (builtin.os.tag == .windows) {
478	return self.inner.next(allocator);	556	const cmd_line_w = os.windows.kernel32.GetCommandLineW();
479	} else {	557	return ArgIterator{ .inner = try InnerType.initUtf16le(allocator, cmd_line_w) };
480	return try allocator.dupeZ(u8, self.inner.next() orelse return null);
481	}	558	}
482	}
483		559
484	/// If you only are targeting posix you can call this and not need an allocator.	560	return ArgIterator{ .inner = InnerType.init() };
485	pub fn nextPosix(self: *ArgIterator) ?[:0]const u8 {
486	return self.inner.next();
487	}	561	}
488		562
489	/// If you only are targeting WASI, you can call this and not need an allocator.	563	/// Get the next argument. Returns 'null' if we are at the end.
490	pub fn nextWasi(self: *ArgIterator) ?[:0]const u8 {	564	/// Returned slice is pointing to the iterator's internal buffer.
		565	pub fn next(self: *ArgIterator) ?([:0]const u8) {
491	return self.inner.next();	566	return self.inner.next();
492	}	567	}
493		568
...	@@ -500,13 +575,18 @@ pub const ArgIterator = struct {	...	@@ -500,13 +575,18 @@ pub const ArgIterator = struct {
500	/// Call this to free the iterator's internal buffer if the iterator	575	/// Call this to free the iterator's internal buffer if the iterator
501	/// was created with `initWithAllocator` function.	576	/// was created with `initWithAllocator` function.
502	pub fn deinit(self: *ArgIterator) void {	577	pub fn deinit(self: *ArgIterator) void {
503	// Unless we're targeting WASI, this is a no-op.	578	// Unless we're targeting WASI or Windows, this is a no-op.
504	if (builtin.os.tag == .wasi and !builtin.link_libc) {	579	if (builtin.os.tag == .wasi and !builtin.link_libc) {
505	self.inner.deinit();	580	self.inner.deinit();
506	}	581	}
		582
		583	if (builtin.os.tag == .windows) {
		584	self.inner.deinit();
		585	}
507	}	586	}
508	};	587	};
509		588
		589	/// Use argsWithAllocator() for cross-platform code
510	pub fn args() ArgIterator {	590	pub fn args() ArgIterator {
511	return ArgIterator.init();	591	return ArgIterator.init();
512	}	592	}
...	@@ -518,12 +598,10 @@ pub fn argsWithAllocator(allocator: mem.Allocator) ArgIterator.InitError!ArgIter	...	@@ -518,12 +598,10 @@ pub fn argsWithAllocator(allocator: mem.Allocator) ArgIterator.InitError!ArgIter
518		598
519	test "args iterator" {	599	test "args iterator" {
520	var ga = std.testing.allocator;	600	var ga = std.testing.allocator;
521	var it = if (builtin.os.tag == .wasi) try argsWithAllocator(ga) else args();	601	var it = try argsWithAllocator(ga);
522	defer it.deinit(); // no-op unless WASI	602	defer it.deinit(); // no-op unless WASI or Windows
523
524	const prog_name = (try it.next(ga)) orelse unreachable;
525	defer ga.free(prog_name);
526		603
		604	const prog_name = it.next() orelse unreachable;
527	const expected_suffix = switch (builtin.os.tag) {	605	const expected_suffix = switch (builtin.os.tag) {
528	.wasi => "test.wasm",	606	.wasi => "test.wasm",
529	.windows => "test.exe",	607	.windows => "test.exe",
...	@@ -533,14 +611,14 @@ test "args iterator" {	...	@@ -533,14 +611,14 @@ test "args iterator" {
533		611
534	try testing.expect(mem.eql(u8, expected_suffix, given_suffix));	612	try testing.expect(mem.eql(u8, expected_suffix, given_suffix));
535	try testing.expect(it.skip()); // Skip over zig_exe_path, passed to the test runner	613	try testing.expect(it.skip()); // Skip over zig_exe_path, passed to the test runner
536	try testing.expect((try it.next(ga)) == null);	614	try testing.expect(it.next() == null);
537	try testing.expect(!it.skip());	615	try testing.expect(!it.skip());
538	}	616	}
539		617
540	/// Caller must call argsFree on result.	618	/// Caller must call argsFree on result.
541	pub fn argsAlloc(allocator: mem.Allocator) ![][:0]u8 {	619	pub fn argsAlloc(allocator: mem.Allocator) ![][:0]u8 {
542	// TODO refactor to only make 1 allocation.	620	// TODO refactor to only make 1 allocation.
543	var it = if (builtin.os.tag == .wasi) try argsWithAllocator(allocator) else args();	621	var it = try argsWithAllocator(allocator);
544	defer it.deinit();	622	defer it.deinit();
545		623
546	var contents = std.ArrayList(u8).init(allocator);	624	var contents = std.ArrayList(u8).init(allocator);
...	@@ -549,8 +627,7 @@ pub fn argsAlloc(allocator: mem.Allocator) ![][:0]u8 {	...	@@ -549,8 +627,7 @@ pub fn argsAlloc(allocator: mem.Allocator) ![][:0]u8 {
549	var slice_list = std.ArrayList(usize).init(allocator);	627	var slice_list = std.ArrayList(usize).init(allocator);
550	defer slice_list.deinit();	628	defer slice_list.deinit();
551		629
552	while (try it.next(allocator)) \|arg\| {	630	while (it.next()) \|arg\| {
553	defer allocator.free(arg);
554	try contents.appendSlice(arg[0 .. arg.len + 1]);	631	try contents.appendSlice(arg[0 .. arg.len + 1]);
555	try slice_list.append(arg.len);	632	try slice_list.append(arg.len);
556	}	633	}
...	@@ -586,16 +663,17 @@ pub fn argsFree(allocator: mem.Allocator, args_alloc: []const [:0]u8) void {	...	@@ -586,16 +663,17 @@ pub fn argsFree(allocator: mem.Allocator, args_alloc: []const [:0]u8) void {
586	return allocator.free(aligned_allocated_buf);	663	return allocator.free(aligned_allocated_buf);
587	}	664	}
588		665
589	test "windows arg parsing" {	666	test "general arg parsing" {
590	const utf16Literal = std.unicode.utf8ToUtf16LeStringLiteral;	667	try testGeneralCmdLine("a b\tc d", &[_][]const u8{ "a", "b", "c", "d" });
591	try testWindowsCmdLine(utf16Literal("a b\tc d"), &[_][]const u8{ "a", "b", "c", "d" });	668	try testGeneralCmdLine("\"abc\" d e", &[_][]const u8{ "abc", "d", "e" });
592	try testWindowsCmdLine(utf16Literal("\"abc\" d e"), &[_][]const u8{ "abc", "d", "e" });	669	try testGeneralCmdLine("a\\\\\\b d\"e f\"g h", &[_][]const u8{ "a\\\\\\b", "de fg", "h" });
593	try testWindowsCmdLine(utf16Literal("a\\\\\\b d\"e f\"g h"), &[_][]const u8{ "a\\\\\\b", "de fg", "h" });	670	try testGeneralCmdLine("a\\\\\\\"b c d", &[_][]const u8{ "a\\\"b", "c", "d" });
594	try testWindowsCmdLine(utf16Literal("a\\\\\\\"b c d"), &[_][]const u8{ "a\\\"b", "c", "d" });	671	try testGeneralCmdLine("a\\\\\\\\\"b c\" d e", &[_][]const u8{ "a\\\\b c", "d", "e" });
595	try testWindowsCmdLine(utf16Literal("a\\\\\\\\\"b c\" d e"), &[_][]const u8{ "a\\\\b c", "d", "e" });	672	try testGeneralCmdLine("a b\tc \"d f", &[_][]const u8{ "a", "b", "c", "d f" });
596	try testWindowsCmdLine(utf16Literal("a b\tc \"d f"), &[_][]const u8{ "a", "b", "c", "d f" });	673	try testGeneralCmdLine("j k l\\", &[_][]const u8{ "j", "k", "l\\" });
597		674	try testGeneralCmdLine("\"\" x y z\\\\", &[_][]const u8{ "", "x", "y", "z\\\\" });
598	try testWindowsCmdLine(utf16Literal("\".\\..\\zig-cache\\build\" \"bin\\zig.exe\" \".\\..\" \".\\..\\zig-cache\" \"--help\""), &[_][]const u8{	675
		676	try testGeneralCmdLine("\".\\..\\zig-cache\\build\" \"bin\\zig.exe\" \".\\..\" \".\\..\\zig-cache\" \"--help\"", &[_][]const u8{
599	".\\..\\zig-cache\\build",	677	".\\..\\zig-cache\\build",
600	"bin\\zig.exe",	678	"bin\\zig.exe",
601	".\\..",	679	".\\..",
...	@@ -604,14 +682,52 @@ test "windows arg parsing" {	...	@@ -604,14 +682,52 @@ test "windows arg parsing" {
604	});	682	});
605	}	683	}
606		684
607	fn testWindowsCmdLine(input_cmd_line: [*]const u16, expected_args: []const []const u8) !void {	685	fn testGeneralCmdLine(input_cmd_line: []const u8, expected_args: []const []const u8) !void {
608	var it = ArgIteratorWindows.initWithCmdLine(input_cmd_line);	686	var it = try ArgIteratorGeneral(.{ .comments_supported = false })
		687	.init(std.testing.allocator, input_cmd_line);
		688	defer it.deinit();
		689	for (expected_args) \|expected_arg\| {
		690	const arg = it.next().?;
		691	try testing.expectEqualStrings(expected_arg, arg);
		692	}
		693	try testing.expect(it.next() == null);
		694	}
		695
		696	test "response file arg parsing" {
		697	try testResponseFileCmdLine(
		698	\\a b
		699	\\c d\
		700	, &[_][]const u8{ "a", "b", "c", "d\\" });
		701	try testResponseFileCmdLine("a b c d\\", &[_][]const u8{ "a", "b", "c", "d\\" });
		702
		703	try testResponseFileCmdLine(
		704	\\j
		705	\\ k l # this is a comment \\ \\\ \\\\ "none" "\\" "\\\"
		706	\\ "m" #another comment
		707	\\
		708	, &[_][]const u8{ "j", "k", "l", "m" });
		709
		710	try testResponseFileCmdLine(
		711	\\ "" q ""
		712	\\ "r s # t" "u\" v" #another comment
		713	\\
		714	, &[_][]const u8{ "", "q", "", "r s # t", "u\" v" });
		715
		716	try testResponseFileCmdLine(
		717	\\ -l"advapi32" a# b#c d#
		718	\\e\\\
		719	, &[_][]const u8{ "-ladvapi32", "a#", "b#c", "d#", "e\\\\\\" });
		720	}
		721
		722	fn testResponseFileCmdLine(input_cmd_line: []const u8, expected_args: []const []const u8) !void {
		723	var it = try ArgIteratorGeneral(.{ .comments_supported = true })
		724	.init(std.testing.allocator, input_cmd_line);
		725	defer it.deinit();
609	for (expected_args) \|expected_arg\| {	726	for (expected_args) \|expected_arg\| {
610	const arg = (it.next(std.testing.allocator) catch unreachable).?;	727	const arg = it.next().?;
611	defer std.testing.allocator.free(arg);
612	try testing.expectEqualStrings(expected_arg, arg);	728	try testing.expectEqualStrings(expected_arg, arg);
613	}	729	}
614	try testing.expect((try it.next(std.testing.allocator)) == null);	730	try testing.expect(it.next() == null);
615	}	731	}
616		732
617	pub const UserInfo = struct {	733	pub const UserInfo = struct {

lib/std/unicode.zig+2-2

...	@@ -568,8 +568,8 @@ pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8	...	@@ -568,8 +568,8 @@ pub fn utf16leToUtf8Alloc(allocator: mem.Allocator, utf16le: []const u16) ![]u8
568		568
569	/// Caller must free returned memory.	569	/// Caller must free returned memory.
570	pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0]u8 {	570	pub fn utf16leToUtf8AllocZ(allocator: mem.Allocator, utf16le: []const u16) ![:0]u8 {
571	// optimistically guess that it will all be ascii.	571	// optimistically guess that it will all be ascii (and allocate space for the null terminator)
572	var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);	572	var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len + 1);
573	errdefer result.deinit();	573	errdefer result.deinit();
574	var out_index: usize = 0;	574	var out_index: usize = 0;
575	var it = Utf16LeIterator.init(utf16le);	575	var it = Utf16LeIterator.init(utf16le);

src/main.zig+32-24

...	@@ -4148,7 +4148,8 @@ pub const ClangArgIterator = struct {	...	@@ -4148,7 +4148,8 @@ pub const ClangArgIterator = struct {
4148	argv: []const []const u8,	4148	argv: []const []const u8,
4149	next_index: usize,	4149	next_index: usize,
4150	root_args: ?*Args,	4150	root_args: ?*Args,
4151	allocator: Allocator,	4151	arg_iterator_response_file: ArgIteratorResponseFile,
		4152	arena: Allocator,
4152		4153
4153	pub const ZigEquivalent = enum {	4154	pub const ZigEquivalent = enum {
4154	target,	4155	target,
...	@@ -4210,7 +4211,7 @@ pub const ClangArgIterator = struct {	...	@@ -4210,7 +4211,7 @@ pub const ClangArgIterator = struct {
4210	argv: []const []const u8,	4211	argv: []const []const u8,
4211	};	4212	};
4212		4213
4213	fn init(allocator: Allocator, argv: []const []const u8) ClangArgIterator {	4214	fn init(arena: Allocator, argv: []const []const u8) ClangArgIterator {
4214	return .{	4215	return .{
4215	.next_index = 2, // `zig cc foo` this points to `foo`	4216	.next_index = 2, // `zig cc foo` this points to `foo`
4216	.has_next = argv.len > 2,	4217	.has_next = argv.len > 2,
...	@@ -4220,10 +4221,22 @@ pub const ClangArgIterator = struct {	...	@@ -4220,10 +4221,22 @@ pub const ClangArgIterator = struct {
4220	.other_args = undefined,	4221	.other_args = undefined,
4221	.argv = argv,	4222	.argv = argv,
4222	.root_args = null,	4223	.root_args = null,
4223	.allocator = allocator,	4224	.arg_iterator_response_file = undefined,
		4225	.arena = arena,
4224	};	4226	};
4225	}	4227	}
4226		4228
		4229	const ArgIteratorResponseFile = process.ArgIteratorGeneral(.{ .comments_supported = true });
		4230
		4231	/// Initialize the arguments from a Response File. "*.rsp"
		4232	fn initArgIteratorResponseFile(allocator: Allocator, resp_file_path: []const u8) !ArgIteratorResponseFile {
		4233	const max_bytes = 10 * 1024 * 1024; // 10 MiB of command line arguments is a reasonable limit
		4234	var cmd_line = try fs.cwd().readFileAlloc(allocator, resp_file_path, max_bytes);
		4235	errdefer allocator.free(cmd_line);
		4236
		4237	return ArgIteratorResponseFile.initTakeOwnership(allocator, cmd_line);
		4238	}
		4239
4227	fn next(self: *ClangArgIterator) !void {	4240	fn next(self: *ClangArgIterator) !void {
4228	assert(self.has_next);	4241	assert(self.has_next);
4229	assert(self.next_index < self.argv.len);	4242	assert(self.next_index < self.argv.len);
...	@@ -4239,31 +4252,25 @@ pub const ClangArgIterator = struct {	...	@@ -4239,31 +4252,25 @@ pub const ClangArgIterator = struct {
4239		4252
4240	// This is a "compiler response file". We must parse the file and treat its	4253	// This is a "compiler response file". We must parse the file and treat its
4241	// contents as command line parameters.	4254	// contents as command line parameters.
4242	const allocator = self.allocator;	4255	const arena = self.arena;
4243	const max_bytes = 10 * 1024 * 1024; // 10 MiB of command line arguments is a reasonable limit
4244	const resp_file_path = arg[1..];	4256	const resp_file_path = arg[1..];
4245	const resp_contents = fs.cwd().readFileAlloc(allocator, resp_file_path, max_bytes) catch \|err\| {	4257
		4258	self.arg_iterator_response_file =
		4259	initArgIteratorResponseFile(arena, resp_file_path) catch \|err\| {
4246	fatal("unable to read response file '{s}': {s}", .{ resp_file_path, @errorName(err) });	4260	fatal("unable to read response file '{s}': {s}", .{ resp_file_path, @errorName(err) });
4247	};	4261	};
4248	defer allocator.free(resp_contents);	4262	// NOTE: The ArgIteratorResponseFile returns tokens from next() that are slices of an
4249	// TODO is there a specification for this file format? Let's find it and make this parsing more robust	4263	// internal buffer. This internal buffer is arena allocated, so it is not cleaned up here.
4250	// at the very least I'm guessing this needs to handle quotes and `#` comments.	4264
4251	var it = mem.tokenize(u8, resp_contents, " \t\r\n");	4265	var resp_arg_list = std.ArrayList([]const u8).init(arena);
4252	var resp_arg_list = std.ArrayList([]const u8).init(allocator);
4253	defer resp_arg_list.deinit();	4266	defer resp_arg_list.deinit();
4254	{	4267	{
4255	errdefer {	4268	while (self.arg_iterator_response_file.next()) \|token\| {
4256	for (resp_arg_list.items) \|item\| {	4269	try resp_arg_list.append(token);
4257	allocator.free(mem.span(item));
4258	}
4259	}	4270	}
4260	while (it.next()) \|token\| {	4271
4261	const dupe_token = try allocator.dupeZ(u8, token);	4272	const args = try arena.create(Args);
4262	errdefer allocator.free(dupe_token);	4273	errdefer arena.destroy(args);
4263	try resp_arg_list.append(dupe_token);
4264	}
4265	const args = try allocator.create(Args);
4266	errdefer allocator.destroy(args);
4267	args.* = .{	4274	args.* = .{
4268	.next_index = self.next_index,	4275	.next_index = self.next_index,
4269	.argv = self.argv,	4276	.argv = self.argv,
...	@@ -4284,6 +4291,7 @@ pub const ClangArgIterator = struct {	...	@@ -4284,6 +4291,7 @@ pub const ClangArgIterator = struct {
4284	arg = mem.span(self.argv[self.next_index]);	4291	arg = mem.span(self.argv[self.next_index]);
4285	self.incrementArgIndex();	4292	self.incrementArgIndex();
4286	}	4293	}
		4294
4287	if (mem.eql(u8, arg, "-") or !mem.startsWith(u8, arg, "-")) {	4295	if (mem.eql(u8, arg, "-") or !mem.startsWith(u8, arg, "-")) {
4288	self.zig_equivalent = .positional;	4296	self.zig_equivalent = .positional;
4289	self.only_arg = arg;	4297	self.only_arg = arg;
...	@@ -4383,13 +4391,13 @@ pub const ClangArgIterator = struct {	...	@@ -4383,13 +4391,13 @@ pub const ClangArgIterator = struct {
4383	}	4391	}
4384		4392
4385	fn resolveRespFileArgs(self: *ClangArgIterator) void {	4393	fn resolveRespFileArgs(self: *ClangArgIterator) void {
4386	const allocator = self.allocator;	4394	const arena = self.arena;
4387	if (self.next_index >= self.argv.len) {	4395	if (self.next_index >= self.argv.len) {
4388	if (self.root_args) \|root_args\| {	4396	if (self.root_args) \|root_args\| {
4389	self.next_index = root_args.next_index;	4397	self.next_index = root_args.next_index;
4390	self.argv = root_args.argv;	4398	self.argv = root_args.argv;
4391		4399
4392	allocator.destroy(root_args);	4400	arena.destroy(root_args);
4393	self.root_args = null;	4401	self.root_args = null;
4394	}	4402	}
4395	if (self.next_index >= self.argv.len) {	4403	if (self.next_index >= self.argv.len) {

test/cli.zig+4-5

 @@ -11,18 +11,17 @@ pub fn main() !void {
     var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
     defer arena.deinit();
-    var arg_it = process.args();
+    a = arena.allocator();
+    var arg_it = try process.argsWithAllocator(a);
     // skip my own exe name
     _ = arg_it.skip();
-    a = arena.allocator();
+    const zig_exe_rel = arg_it.next() orelse {
-    const zig_exe_rel = (try arg_it.next(a)) orelse {
         std.debug.print("Expected first argument to be path to zig compiler\n", .{});
         return error.InvalidArgs;
     };
-    const cache_root = (try arg_it.next(a)) orelse {
+    const cache_root = arg_it.next() orelse {
         std.debug.print("Expected second argument to be cache root directory path\n", .{});
         return error.InvalidArgs;
     };

test/compare_output.zig+11-5

 @@ -291,7 +291,9 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
         \\    stdout.print("before\n", .{}) catch unreachable;
         \\    defer stdout.print("defer1\n", .{}) catch unreachable;
         \\    defer stdout.print("defer2\n", .{}) catch unreachable;
-        \\    var args_it = @import("std").process.args();
+        \\    var arena = @import("std").heap.ArenaAllocator.init(@import("std").testing.allocator);
+        \\    defer arena.deinit();
+        \\    var args_it = @import("std").process.argsWithAllocator(arena.allocator()) catch unreachable;
         \\    if (args_it.skip() and !args_it.skip()) return;
         \\    defer stdout.print("defer3\n", .{}) catch unreachable;
         \\    stdout.print("after\n", .{}) catch unreachable;
 @@ -358,11 +360,13 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\const allocator = std.testing.allocator;
             \\
             \\pub fn main() !void {
-            \\    var args_it = std.process.args();
+            \\    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+            \\    defer arena.deinit();
+            \\    var args_it = try std.process.argsWithAllocator(arena.allocator());
             \\    const stdout = io.getStdOut().writer();
             \\    var index: usize = 0;
             \\    _ = args_it.skip();
-            \\    while (try args_it.next(allocator)) |arg| : (index += 1) {
+            \\    while (args_it.next()) |arg| : (index += 1) {
             \\        try stdout.print("{}: {s}\n", .{index, arg});
             \\    }
             \\}
 @@ -396,11 +400,13 @@ pub fn addCases(cases: *tests.CompareOutputContext) void {
             \\const allocator = std.testing.allocator;
             \\
             \\pub fn main() !void {
-            \\    var args_it = std.process.args();
+            \\    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+            \\    defer arena.deinit();
+            \\    var args_it = try std.process.argsWithAllocator(arena.allocator());
             \\    const stdout = io.getStdOut().writer();
             \\    var index: usize = 0;
             \\    _ = args_it.skip();
-            \\    while (try args_it.next(allocator)) |arg| : (index += 1) {
+            \\    while (args_it.next()) |arg| : (index += 1) {
             \\        try stdout.print("{}: {s}\n", .{index, arg});
             \\    }
             \\}