authorgravatar for Rageoholic@users.noreply.github.comRageoholic <Rageoholic@users.noreply.github.com> 2020-11-30 10:47:01-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-11-30 13:47:01-05:00
log0369b65082be49eefacf767774a3d40ad7706de7
tree6cd736f8eca1b99f4e6746e159ed46cbb3eaa156
parentb8f09f773aa6f5133b9f756b872525a0f550ac17
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Switch to using unicode when parsing the command line on windows (#7241)

* Switch to using unicode when parsing the command line on windows * Apply changes by LemonBoy and *hopefully* fix tests on MIPs Co-authored-by: LemonBoy <LemonBoy@users.noreply.github.com> * Fix up next and skip * Move comment to more relevant place Co-authored-by: LemonBoy <LemonBoy@users.noreply.github.com>

2 files changed, 77 insertions(+), 35 deletions(-)

lib/std/process.zig+50-29
......@@ -285,27 +285,35 @@ pub const ArgIteratorWasi = struct {
285285
286286pub const ArgIteratorWindows = struct {
287287 index: usize,
288 cmd_line: [*]const u8,
288 cmd_line: [*]const u16,
289289
290 pub const NextError = error{OutOfMemory};
290 pub const NextError = error{ OutOfMemory, InvalidCmdLine };
291291
292292 pub fn init() ArgIteratorWindows {
293 return initWithCmdLine(os.windows.kernel32.GetCommandLineA());
293 return initWithCmdLine(os.windows.kernel32.GetCommandLineW());
294294 }
295295
296 pub fn initWithCmdLine(cmd_line: [*]const u8) ArgIteratorWindows {
296 pub fn initWithCmdLine(cmd_line: [*]const u16) ArgIteratorWindows {
297297 return ArgIteratorWindows{
298298 .index = 0,
299299 .cmd_line = cmd_line,
300300 };
301301 }
302302
303 fn getPointAtIndex(self: *ArgIteratorWindows) u16 {
304 // According to
305 // https://docs.microsoft.com/en-us/windows/win32/intl/using-byte-order-marks
306 // Microsoft uses UTF16-LE. So we just read assuming it's little
307 // endian.
308 return std.mem.littleToNative(u16, self.cmd_line[self.index]);
309 }
310
303311 /// You must free the returned memory when done.
304312 pub fn next(self: *ArgIteratorWindows, allocator: *Allocator) ?(NextError![:0]u8) {
305313 // march forward over whitespace
306314 while (true) : (self.index += 1) {
307 const byte = self.cmd_line[self.index];
308 switch (byte) {
315 const character = self.getPointAtIndex();
316 switch (character) {
309317 0 => return null,
310318 ' ', '\t' => continue,
311319 else => break,
......@@ -318,8 +326,8 @@ pub const ArgIteratorWindows = struct {
318326 pub fn skip(self: *ArgIteratorWindows) bool {
319327 // march forward over whitespace
320328 while (true) : (self.index += 1) {
321 const byte = self.cmd_line[self.index];
322 switch (byte) {
329 const character = self.getPointAtIndex();
330 switch (character) {
323331 0 => return false,
324332 ' ', '\t' => continue,
325333 else => break,
......@@ -329,8 +337,8 @@ pub const ArgIteratorWindows = struct {
329337 var backslash_count: usize = 0;
330338 var in_quote = false;
331339 while (true) : (self.index += 1) {
332 const byte = self.cmd_line[self.index];
333 switch (byte) {
340 const character = self.getPointAtIndex();
341 switch (character) {
334342 0 => return true,
335343 '"' => {
336344 const quote_is_real = backslash_count % 2 == 0;
......@@ -356,15 +364,17 @@ pub const ArgIteratorWindows = struct {
356364 }
357365
358366 fn internalNext(self: *ArgIteratorWindows, allocator: *Allocator) NextError![:0]u8 {
359 var buf = try std.ArrayListSentineled(u8, 0).init(allocator, "");
367 var buf = std.ArrayList(u16).init(allocator);
360368 defer buf.deinit();
361369
362370 var backslash_count: usize = 0;
363371 var in_quote = false;
364372 while (true) : (self.index += 1) {
365 const byte = self.cmd_line[self.index];
366 switch (byte) {
367 0 => return buf.toOwnedSlice(),
373 const character = self.getPointAtIndex();
374 switch (character) {
375 0 => {
376 return convertFromWindowsCmdLineToUTF8(allocator, buf.items);
377 },
368378 '"' => {
369379 const quote_is_real = backslash_count % 2 == 0;
370380 try self.emitBackslashes(&buf, backslash_count / 2);
......@@ -373,7 +383,7 @@ pub const ArgIteratorWindows = struct {
373383 if (quote_is_real) {
374384 in_quote = !in_quote;
375385 } else {
376 try buf.append('"');
386 try buf.append(std.mem.nativeToLittle(u16, '"'));
377387 }
378388 },
379389 '\\' => {
......@@ -383,24 +393,34 @@ pub const ArgIteratorWindows = struct {
383393 try self.emitBackslashes(&buf, backslash_count);
384394 backslash_count = 0;
385395 if (in_quote) {
386 try buf.append(byte);
396 try buf.append(std.mem.nativeToLittle(u16, character));
387397 } else {
388 return buf.toOwnedSlice();
398 return convertFromWindowsCmdLineToUTF8(allocator, buf.items);
389399 }
390400 },
391401 else => {
392402 try self.emitBackslashes(&buf, backslash_count);
393403 backslash_count = 0;
394 try buf.append(byte);
404 try buf.append(std.mem.nativeToLittle(u16, character));
395405 },
396406 }
397407 }
398408 }
399409
400 fn emitBackslashes(self: *ArgIteratorWindows, buf: *std.ArrayListSentineled(u8, 0), emit_count: usize) !void {
410 fn convertFromWindowsCmdLineToUTF8(allocator: *Allocator, buf: []u16) NextError![:0]u8 {
411 return std.unicode.utf16leToUtf8AllocZ(allocator, buf) catch |err| switch (err) {
412 error.ExpectedSecondSurrogateHalf,
413 error.DanglingSurrogateHalf,
414 error.UnexpectedSecondSurrogateHalf,
415 => return error.InvalidCmdLine,
416
417 error.OutOfMemory => return error.OutOfMemory,
418 };
419 }
420 fn emitBackslashes(self: *ArgIteratorWindows, buf: *std.ArrayList(u16), emit_count: usize) !void {
401421 var i: usize = 0;
402422 while (i < emit_count) : (i += 1) {
403 try buf.append('\\');
423 try buf.append(std.mem.nativeToLittle(u16, '\\'));
404424 }
405425 }
406426};
......@@ -552,14 +572,15 @@ pub fn argsFree(allocator: *mem.Allocator, args_alloc: []const [:0]u8) void {
552572}
553573
554574test "windows arg parsing" {
555 testWindowsCmdLine("a b\tc d", &[_][]const u8{ "a", "b", "c", "d" });
556 testWindowsCmdLine("\"abc\" d e", &[_][]const u8{ "abc", "d", "e" });
557 testWindowsCmdLine("a\\\\\\b d\"e f\"g h", &[_][]const u8{ "a\\\\\\b", "de fg", "h" });
558 testWindowsCmdLine("a\\\\\\\"b c d", &[_][]const u8{ "a\\\"b", "c", "d" });
559 testWindowsCmdLine("a\\\\\\\\\"b c\" d e", &[_][]const u8{ "a\\\\b c", "d", "e" });
560 testWindowsCmdLine("a b\tc \"d f", &[_][]const u8{ "a", "b", "c", "d f" });
561
562 testWindowsCmdLine("\".\\..\\zig-cache\\build\" \"bin\\zig.exe\" \".\\..\" \".\\..\\zig-cache\" \"--help\"", &[_][]const u8{
575 const utf16Literal = std.unicode.utf8ToUtf16LeStringLiteral;
576 testWindowsCmdLine(utf16Literal("a b\tc d"), &[_][]const u8{ "a", "b", "c", "d" });
577 testWindowsCmdLine(utf16Literal("\"abc\" d e"), &[_][]const u8{ "abc", "d", "e" });
578 testWindowsCmdLine(utf16Literal("a\\\\\\b d\"e f\"g h"), &[_][]const u8{ "a\\\\\\b", "de fg", "h" });
579 testWindowsCmdLine(utf16Literal("a\\\\\\\"b c d"), &[_][]const u8{ "a\\\"b", "c", "d" });
580 testWindowsCmdLine(utf16Literal("a\\\\\\\\\"b c\" d e"), &[_][]const u8{ "a\\\\b c", "d", "e" });
581 testWindowsCmdLine(utf16Literal("a b\tc \"d f"), &[_][]const u8{ "a", "b", "c", "d f" });
582
583 testWindowsCmdLine(utf16Literal("\".\\..\\zig-cache\\build\" \"bin\\zig.exe\" \".\\..\" \".\\..\\zig-cache\" \"--help\""), &[_][]const u8{
563584 ".\\..\\zig-cache\\build",
564585 "bin\\zig.exe",
565586 ".\\..",
......@@ -568,7 +589,7 @@ test "windows arg parsing" {
568589 });
569590}
570591
571fn testWindowsCmdLine(input_cmd_line: [*]const u8, expected_args: []const []const u8) void {
592fn testWindowsCmdLine(input_cmd_line: [*]const u16, expected_args: []const []const u8) void {
572593 var it = ArgIteratorWindows.initWithCmdLine(input_cmd_line);
573594 for (expected_args) |expected_arg| {
574595 const arg = it.next(std.testing.allocator).? catch unreachable;
lib/std/unicode.zig+27-6
......@@ -25,10 +25,10 @@ pub fn utf8CodepointSequenceLength(c: u21) !u3 {
2525pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
2626 // The switch is optimized much better than a "smart" approach using @clz
2727 return switch (first_byte) {
28 0b0000_0000 ... 0b0111_1111 => 1,
29 0b1100_0000 ... 0b1101_1111 => 2,
30 0b1110_0000 ... 0b1110_1111 => 3,
31 0b1111_0000 ... 0b1111_0111 => 4,
28 0b0000_0000...0b0111_1111 => 1,
29 0b1100_0000...0b1101_1111 => 2,
30 0b1110_0000...0b1110_1111 => 3,
31 0b1111_0000...0b1111_0111 => 4,
3232 else => error.Utf8InvalidStartByte,
3333 };
3434}
......@@ -157,8 +157,8 @@ pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
157157/// Returns true if the given unicode codepoint can be encoded in UTF-8.
158158pub fn utf8ValidCodepoint(value: u21) bool {
159159 return switch (value) {
160 0xD800 ... 0xDFFF => false, // Surrogates range
161 0x110000 ... 0x1FFFFF => false, // Above the maximum codepoint value
160 0xD800...0xDFFF => false, // Surrogates range
161 0x110000...0x1FFFFF => false, // Above the maximum codepoint value
162162 else => true,
163163 };
164164}
......@@ -574,6 +574,27 @@ pub fn utf16leToUtf8Alloc(allocator: *mem.Allocator, utf16le: []const u16) ![]u8
574574 return result.toOwnedSlice();
575575}
576576
577/// Caller must free returned memory.
578pub fn utf16leToUtf8AllocZ(allocator: *mem.Allocator, utf16le: []const u16) ![:0]u8 {
579 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);
580 // optimistically guess that it will all be ascii.
581 try result.ensureCapacity(utf16le.len);
582 var out_index: usize = 0;
583 var it = Utf16LeIterator.init(utf16le);
584 while (try it.nextCodepoint()) |codepoint| {
585 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
586 try result.resize(result.items.len + utf8_len);
587 assert((utf8Encode(codepoint, result.items[out_index..]) catch unreachable) == utf8_len);
588 out_index += utf8_len;
589 }
590
591 const len = result.items.len;
592
593 try result.append(0);
594
595 return result.toOwnedSlice()[0..len :0];
596}
597
577598/// Asserts that the output buffer is big enough.
578599/// Returns end byte index into utf8.
579600pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize {