authorgravatar for Rageoholic@users.noreply.github.comRageoholic <Rageoholic@users.noreply.github.com> 2020-11-30 10:47:01-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-11-30 13:47:01-05:00
log0369b65082be49eefacf767774a3d40ad7706de7
tree6cd736f8eca1b99f4e6746e159ed46cbb3eaa156
parentb8f09f773aa6f5133b9f756b872525a0f550ac17
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Switch to using unicode when parsing the command line on windows (#7241)

* Switch to using unicode when parsing the command line on windows * Apply changes by LemonBoy and *hopefully* fix tests on MIPs Co-authored-by: LemonBoy <LemonBoy@users.noreply.github.com> * Fix up next and skip * Move comment to more relevant place Co-authored-by: LemonBoy <LemonBoy@users.noreply.github.com>

2 files changed, 77 insertions(+), 35 deletions(-)

lib/std/process.zig+50-29
...@@ -285,27 +285,35 @@ pub const ArgIteratorWasi = struct {...@@ -285,27 +285,35 @@ pub const ArgIteratorWasi = struct {
285285
286pub const ArgIteratorWindows = struct {286pub const ArgIteratorWindows = struct {
287 index: usize,287 index: usize,
288 cmd_line: [*]const u8,288 cmd_line: [*]const u16,
289289
290 pub const NextError = error{OutOfMemory};290 pub const NextError = error{ OutOfMemory, InvalidCmdLine };
291291
292 pub fn init() ArgIteratorWindows {292 pub fn init() ArgIteratorWindows {
293 return initWithCmdLine(os.windows.kernel32.GetCommandLineA());293 return initWithCmdLine(os.windows.kernel32.GetCommandLineW());
294 }294 }
295295
296 pub fn initWithCmdLine(cmd_line: [*]const u8) ArgIteratorWindows {296 pub fn initWithCmdLine(cmd_line: [*]const u16) ArgIteratorWindows {
297 return ArgIteratorWindows{297 return ArgIteratorWindows{
298 .index = 0,298 .index = 0,
299 .cmd_line = cmd_line,299 .cmd_line = cmd_line,
300 };300 };
301 }301 }
302302
303 fn getPointAtIndex(self: *ArgIteratorWindows) u16 {
304 // According to
305 // https://docs.microsoft.com/en-us/windows/win32/intl/using-byte-order-marks
306 // Microsoft uses UTF16-LE. So we just read assuming it's little
307 // endian.
308 return std.mem.littleToNative(u16, self.cmd_line[self.index]);
309 }
310
303 /// You must free the returned memory when done.311 /// You must free the returned memory when done.
304 pub fn next(self: *ArgIteratorWindows, allocator: *Allocator) ?(NextError![:0]u8) {312 pub fn next(self: *ArgIteratorWindows, allocator: *Allocator) ?(NextError![:0]u8) {
305 // march forward over whitespace313 // march forward over whitespace
306 while (true) : (self.index += 1) {314 while (true) : (self.index += 1) {
307 const byte = self.cmd_line[self.index];315 const character = self.getPointAtIndex();
308 switch (byte) {316 switch (character) {
309 0 => return null,317 0 => return null,
310 ' ', '\t' => continue,318 ' ', '\t' => continue,
311 else => break,319 else => break,
...@@ -318,8 +326,8 @@ pub const ArgIteratorWindows = struct {...@@ -318,8 +326,8 @@ pub const ArgIteratorWindows = struct {
318 pub fn skip(self: *ArgIteratorWindows) bool {326 pub fn skip(self: *ArgIteratorWindows) bool {
319 // march forward over whitespace327 // march forward over whitespace
320 while (true) : (self.index += 1) {328 while (true) : (self.index += 1) {
321 const byte = self.cmd_line[self.index];329 const character = self.getPointAtIndex();
322 switch (byte) {330 switch (character) {
323 0 => return false,331 0 => return false,
324 ' ', '\t' => continue,332 ' ', '\t' => continue,
325 else => break,333 else => break,
...@@ -329,8 +337,8 @@ pub const ArgIteratorWindows = struct {...@@ -329,8 +337,8 @@ pub const ArgIteratorWindows = struct {
329 var backslash_count: usize = 0;337 var backslash_count: usize = 0;
330 var in_quote = false;338 var in_quote = false;
331 while (true) : (self.index += 1) {339 while (true) : (self.index += 1) {
332 const byte = self.cmd_line[self.index];340 const character = self.getPointAtIndex();
333 switch (byte) {341 switch (character) {
334 0 => return true,342 0 => return true,
335 '"' => {343 '"' => {
336 const quote_is_real = backslash_count % 2 == 0;344 const quote_is_real = backslash_count % 2 == 0;
...@@ -356,15 +364,17 @@ pub const ArgIteratorWindows = struct {...@@ -356,15 +364,17 @@ pub const ArgIteratorWindows = struct {
356 }364 }
357365
358 fn internalNext(self: *ArgIteratorWindows, allocator: *Allocator) NextError![:0]u8 {366 fn internalNext(self: *ArgIteratorWindows, allocator: *Allocator) NextError![:0]u8 {
359 var buf = try std.ArrayListSentineled(u8, 0).init(allocator, "");367 var buf = std.ArrayList(u16).init(allocator);
360 defer buf.deinit();368 defer buf.deinit();
361369
362 var backslash_count: usize = 0;370 var backslash_count: usize = 0;
363 var in_quote = false;371 var in_quote = false;
364 while (true) : (self.index += 1) {372 while (true) : (self.index += 1) {
365 const byte = self.cmd_line[self.index];373 const character = self.getPointAtIndex();
366 switch (byte) {374 switch (character) {
367 0 => return buf.toOwnedSlice(),375 0 => {
376 return convertFromWindowsCmdLineToUTF8(allocator, buf.items);
377 },
368 '"' => {378 '"' => {
369 const quote_is_real = backslash_count % 2 == 0;379 const quote_is_real = backslash_count % 2 == 0;
370 try self.emitBackslashes(&buf, backslash_count / 2);380 try self.emitBackslashes(&buf, backslash_count / 2);
...@@ -373,7 +383,7 @@ pub const ArgIteratorWindows = struct {...@@ -373,7 +383,7 @@ pub const ArgIteratorWindows = struct {
373 if (quote_is_real) {383 if (quote_is_real) {
374 in_quote = !in_quote;384 in_quote = !in_quote;
375 } else {385 } else {
376 try buf.append('"');386 try buf.append(std.mem.nativeToLittle(u16, '"'));
377 }387 }
378 },388 },
379 '\\' => {389 '\\' => {
...@@ -383,24 +393,34 @@ pub const ArgIteratorWindows = struct {...@@ -383,24 +393,34 @@ pub const ArgIteratorWindows = struct {
383 try self.emitBackslashes(&buf, backslash_count);393 try self.emitBackslashes(&buf, backslash_count);
384 backslash_count = 0;394 backslash_count = 0;
385 if (in_quote) {395 if (in_quote) {
386 try buf.append(byte);396 try buf.append(std.mem.nativeToLittle(u16, character));
387 } else {397 } else {
388 return buf.toOwnedSlice();398 return convertFromWindowsCmdLineToUTF8(allocator, buf.items);
389 }399 }
390 },400 },
391 else => {401 else => {
392 try self.emitBackslashes(&buf, backslash_count);402 try self.emitBackslashes(&buf, backslash_count);
393 backslash_count = 0;403 backslash_count = 0;
394 try buf.append(byte);404 try buf.append(std.mem.nativeToLittle(u16, character));
395 },405 },
396 }406 }
397 }407 }
398 }408 }
399409
400 fn emitBackslashes(self: *ArgIteratorWindows, buf: *std.ArrayListSentineled(u8, 0), emit_count: usize) !void {410 fn convertFromWindowsCmdLineToUTF8(allocator: *Allocator, buf: []u16) NextError![:0]u8 {
411 return std.unicode.utf16leToUtf8AllocZ(allocator, buf) catch |err| switch (err) {
412 error.ExpectedSecondSurrogateHalf,
413 error.DanglingSurrogateHalf,
414 error.UnexpectedSecondSurrogateHalf,
415 => return error.InvalidCmdLine,
416
417 error.OutOfMemory => return error.OutOfMemory,
418 };
419 }
420 fn emitBackslashes(self: *ArgIteratorWindows, buf: *std.ArrayList(u16), emit_count: usize) !void {
401 var i: usize = 0;421 var i: usize = 0;
402 while (i < emit_count) : (i += 1) {422 while (i < emit_count) : (i += 1) {
403 try buf.append('\\');423 try buf.append(std.mem.nativeToLittle(u16, '\\'));
404 }424 }
405 }425 }
406};426};
...@@ -552,14 +572,15 @@ pub fn argsFree(allocator: *mem.Allocator, args_alloc: []const [:0]u8) void {...@@ -552,14 +572,15 @@ pub fn argsFree(allocator: *mem.Allocator, args_alloc: []const [:0]u8) void {
552}572}
553573
554test "windows arg parsing" {574test "windows arg parsing" {
555 testWindowsCmdLine("a b\tc d", &[_][]const u8{ "a", "b", "c", "d" });575 const utf16Literal = std.unicode.utf8ToUtf16LeStringLiteral;
556 testWindowsCmdLine("\"abc\" d e", &[_][]const u8{ "abc", "d", "e" });576 testWindowsCmdLine(utf16Literal("a b\tc d"), &[_][]const u8{ "a", "b", "c", "d" });
557 testWindowsCmdLine("a\\\\\\b d\"e f\"g h", &[_][]const u8{ "a\\\\\\b", "de fg", "h" });577 testWindowsCmdLine(utf16Literal("\"abc\" d e"), &[_][]const u8{ "abc", "d", "e" });
558 testWindowsCmdLine("a\\\\\\\"b c d", &[_][]const u8{ "a\\\"b", "c", "d" });578 testWindowsCmdLine(utf16Literal("a\\\\\\b d\"e f\"g h"), &[_][]const u8{ "a\\\\\\b", "de fg", "h" });
559 testWindowsCmdLine("a\\\\\\\\\"b c\" d e", &[_][]const u8{ "a\\\\b c", "d", "e" });579 testWindowsCmdLine(utf16Literal("a\\\\\\\"b c d"), &[_][]const u8{ "a\\\"b", "c", "d" });
560 testWindowsCmdLine("a b\tc \"d f", &[_][]const u8{ "a", "b", "c", "d f" });580 testWindowsCmdLine(utf16Literal("a\\\\\\\\\"b c\" d e"), &[_][]const u8{ "a\\\\b c", "d", "e" });
561581 testWindowsCmdLine(utf16Literal("a b\tc \"d f"), &[_][]const u8{ "a", "b", "c", "d f" });
562 testWindowsCmdLine("\".\\..\\zig-cache\\build\" \"bin\\zig.exe\" \".\\..\" \".\\..\\zig-cache\" \"--help\"", &[_][]const u8{582
583 testWindowsCmdLine(utf16Literal("\".\\..\\zig-cache\\build\" \"bin\\zig.exe\" \".\\..\" \".\\..\\zig-cache\" \"--help\""), &[_][]const u8{
563 ".\\..\\zig-cache\\build",584 ".\\..\\zig-cache\\build",
564 "bin\\zig.exe",585 "bin\\zig.exe",
565 ".\\..",586 ".\\..",
...@@ -568,7 +589,7 @@ test "windows arg parsing" {...@@ -568,7 +589,7 @@ test "windows arg parsing" {
568 });589 });
569}590}
570591
571fn testWindowsCmdLine(input_cmd_line: [*]const u8, expected_args: []const []const u8) void {592fn testWindowsCmdLine(input_cmd_line: [*]const u16, expected_args: []const []const u8) void {
572 var it = ArgIteratorWindows.initWithCmdLine(input_cmd_line);593 var it = ArgIteratorWindows.initWithCmdLine(input_cmd_line);
573 for (expected_args) |expected_arg| {594 for (expected_args) |expected_arg| {
574 const arg = it.next(std.testing.allocator).? catch unreachable;595 const arg = it.next(std.testing.allocator).? catch unreachable;
lib/std/unicode.zig+27-6
...@@ -25,10 +25,10 @@ pub fn utf8CodepointSequenceLength(c: u21) !u3 {...@@ -25,10 +25,10 @@ pub fn utf8CodepointSequenceLength(c: u21) !u3 {
25pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {25pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
26 // The switch is optimized much better than a "smart" approach using @clz26 // The switch is optimized much better than a "smart" approach using @clz
27 return switch (first_byte) {27 return switch (first_byte) {
28 0b0000_0000 ... 0b0111_1111 => 1,28 0b0000_0000...0b0111_1111 => 1,
29 0b1100_0000 ... 0b1101_1111 => 2,29 0b1100_0000...0b1101_1111 => 2,
30 0b1110_0000 ... 0b1110_1111 => 3,30 0b1110_0000...0b1110_1111 => 3,
31 0b1111_0000 ... 0b1111_0111 => 4,31 0b1111_0000...0b1111_0111 => 4,
32 else => error.Utf8InvalidStartByte,32 else => error.Utf8InvalidStartByte,
33 };33 };
34}34}
...@@ -157,8 +157,8 @@ pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {...@@ -157,8 +157,8 @@ pub fn utf8Decode4(bytes: []const u8) Utf8Decode4Error!u21 {
157/// Returns true if the given unicode codepoint can be encoded in UTF-8.157/// Returns true if the given unicode codepoint can be encoded in UTF-8.
158pub fn utf8ValidCodepoint(value: u21) bool {158pub fn utf8ValidCodepoint(value: u21) bool {
159 return switch (value) {159 return switch (value) {
160 0xD800 ... 0xDFFF => false, // Surrogates range160 0xD800...0xDFFF => false, // Surrogates range
161 0x110000 ... 0x1FFFFF => false, // Above the maximum codepoint value161 0x110000...0x1FFFFF => false, // Above the maximum codepoint value
162 else => true,162 else => true,
163 };163 };
164}164}
...@@ -574,6 +574,27 @@ pub fn utf16leToUtf8Alloc(allocator: *mem.Allocator, utf16le: []const u16) ![]u8...@@ -574,6 +574,27 @@ pub fn utf16leToUtf8Alloc(allocator: *mem.Allocator, utf16le: []const u16) ![]u8
574 return result.toOwnedSlice();574 return result.toOwnedSlice();
575}575}
576576
577/// Caller must free returned memory.
578pub fn utf16leToUtf8AllocZ(allocator: *mem.Allocator, utf16le: []const u16) ![:0]u8 {
579 var result = try std.ArrayList(u8).initCapacity(allocator, utf16le.len);
580 // optimistically guess that it will all be ascii.
581 try result.ensureCapacity(utf16le.len);
582 var out_index: usize = 0;
583 var it = Utf16LeIterator.init(utf16le);
584 while (try it.nextCodepoint()) |codepoint| {
585 const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
586 try result.resize(result.items.len + utf8_len);
587 assert((utf8Encode(codepoint, result.items[out_index..]) catch unreachable) == utf8_len);
588 out_index += utf8_len;
589 }
590
591 const len = result.items.len;
592
593 try result.append(0);
594
595 return result.toOwnedSlice()[0..len :0];
596}
597
577/// Asserts that the output buffer is big enough.598/// Asserts that the output buffer is big enough.
578/// Returns end byte index into utf8.599/// Returns end byte index into utf8.
579pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize {600pub fn utf16leToUtf8(utf8: []u8, utf16le: []const u16) !usize {