authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-07-13 16:54:00-07:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2024-07-13 18:19:19-07:00
log1418c8a5d49ecc38a20df07151a7fe942992aea4
tree4386161d9e439aac1b609f8902f0e07e12c44565
parent10914dc31000bc5c3b7164a4101808719d3da07f

ArgIteratorWindows: Store last emitted code unit instead of checking the last 6 emitted bytes

Previously, to ensure args were encoded as well-formed WTF-8 (i.e. no encoded surrogate pairs), the code unit would be encoded and then the last 6 emitted bytes would be checked to see if they were a surrogate pair, and this was done for any emitted code unit (although this was not necessary, it should have only been done when emitting a low surrogate). After this commit, we still want to ensure well-formed WTF-8, but, to do so, the last emitted code point is stored, meaning we can just directly check that the last code unit is a high surrogate and the current code unit is a low surrogate to determine if we have a surrogate pair. This provides some performance benefit over and above a "use the same strategy as before but only check when we're emitting a low surrogate" implementation: Benchmark 1 (111 runs): benchargv-master.exe measurement mean ± σ min … max outliers delta wall_time 45.2ms ± 532us 44.5ms … 49.4ms 2 ( 2%) 0% peak_rss 6.49MB ± 3.94KB 6.46MB … 6.49MB 10 ( 9%) 0% Benchmark 2 (154 runs): benchargv-storelast.exe measurement mean ± σ min … max outliers delta wall_time 32.6ms ± 293us 32.2ms … 34.2ms 8 ( 5%) ⚡- 27.8% ± 0.2% peak_rss 6.49MB ± 5.15KB 6.46MB … 6.49MB 15 (10%) - 0.0% ± 0.0% Benchmark 3 (131 runs): benchargv-onlylow.exe measurement mean ± σ min … max outliers delta wall_time 38.4ms ± 257us 37.9ms … 39.6ms 5 ( 4%) ⚡- 15.1% ± 0.2% peak_rss 6.49MB ± 5.70KB 6.46MB … 6.49MB 9 ( 7%) - 0.0% ± 0.0%

1 files changed, 50 insertions(+), 44 deletions(-)

lib/std/process.zig+50-44
......@@ -717,14 +717,20 @@ pub const ArgIteratorWindows = struct {
717717
718718 const eof = null;
719719
720 fn emitBackslashes(self: *ArgIteratorWindows, count: usize) void {
721 for (0..count) |_| emitCharacter(self, '\\');
720 /// Returns '\' if any backslashes are emitted, otherwise returns `last_emitted_code_unit`.
721 fn emitBackslashes(self: *ArgIteratorWindows, count: usize, last_emitted_code_unit: ?u16) ?u16 {
722 for (0..count) |_| {
723 self.buffer[self.end] = '\\';
724 self.end += 1;
725 }
726 return if (count != 0) '\\' else last_emitted_code_unit;
722727 }
723728
724 fn emitCharacter(self: *ArgIteratorWindows, code_unit: u16) void {
725 const wtf8_len = std.unicode.wtf8Encode(code_unit, self.buffer[self.end..]) catch unreachable;
726 self.end += wtf8_len;
727
729 /// If `last_emitted_code_unit` and `code_unit` form a surrogate pair, then
730 /// the previously emitted high surrogate is overwritten by the codepoint encoded
731 /// by the surrogate pair, and `null` is returned.
732 /// Otherwise, `code_unit` is emitted and returned.
733 fn emitCharacter(self: *ArgIteratorWindows, code_unit: u16, last_emitted_code_unit: ?u16) ?u16 {
728734 // Because we are emitting WTF-8, we need to
729735 // check to see if we've emitted two consecutive surrogate
730736 // codepoints that form a valid surrogate pair in order
......@@ -745,28 +751,24 @@ pub const ArgIteratorWindows = struct {
745751 // and emit the codepoint it encodes, which in this
746752 // example is U+10437 (𐐷), which is encoded in UTF-8 as:
747753 // <0xF0><0x90><0x90><0xB7>
748 concatSurrogatePair(self);
749 }
750
751 fn concatSurrogatePair(self: *ArgIteratorWindows) void {
752 // Surrogate codepoints are always encoded as 3 bytes, so there
753 // must be 6 bytes for a surrogate pair to exist.
754 if (self.end - self.start >= 6) {
755 const window = self.buffer[self.end - 6 .. self.end];
756 const view = unicode.Wtf8View.init(window) catch return;
757 var it = view.iterator();
758 var pair: [2]u16 = undefined;
759 pair[0] = std.mem.nativeToLittle(u16, std.math.cast(u16, it.nextCodepoint().?) orelse return);
760 if (!unicode.utf16IsHighSurrogate(std.mem.littleToNative(u16, pair[0]))) return;
761 pair[1] = std.mem.nativeToLittle(u16, std.math.cast(u16, it.nextCodepoint().?) orelse return);
762 if (!unicode.utf16IsLowSurrogate(std.mem.littleToNative(u16, pair[1]))) return;
763 // We know we have a valid surrogate pair, so convert
764 // it to UTF-8, overwriting the surrogate pair's bytes
765 // and then chop off the extra bytes.
766 const len = unicode.utf16LeToUtf8(window, &pair) catch unreachable;
767 const delta = 6 - len;
768 self.end -= delta;
754 if (last_emitted_code_unit != null and
755 std.unicode.utf16IsLowSurrogate(code_unit) and
756 std.unicode.utf16IsHighSurrogate(last_emitted_code_unit.?))
757 {
758 const codepoint = std.unicode.utf16DecodeSurrogatePair(&.{ last_emitted_code_unit.?, code_unit }) catch unreachable;
759
760 // Unpaired surrogate is 3 bytes long
761 const dest = self.buffer[self.end - 3 ..];
762 const len = unicode.utf8Encode(codepoint, dest) catch unreachable;
763 // All codepoints that require a surrogate pair (> U+FFFF) are encoded as 4 bytes
764 assert(len == 4);
765 self.end += 1;
766 return null;
769767 }
768
769 const wtf8_len = std.unicode.wtf8Encode(code_unit, self.buffer[self.end..]) catch unreachable;
770 self.end += wtf8_len;
771 return code_unit;
770772 }
771773
772774 fn yieldArg(self: *ArgIteratorWindows) [:0]const u8 {
......@@ -783,9 +785,13 @@ pub const ArgIteratorWindows = struct {
783785
784786 const eof = false;
785787
786 fn emitBackslashes(_: *ArgIteratorWindows, _: usize) void {}
788 fn emitBackslashes(_: *ArgIteratorWindows, _: usize, last_emitted_code_unit: ?u16) ?u16 {
789 return last_emitted_code_unit;
790 }
787791
788 fn emitCharacter(_: *ArgIteratorWindows, _: u16) void {}
792 fn emitCharacter(_: *ArgIteratorWindows, _: u16, last_emitted_code_unit: ?u16) ?u16 {
793 return last_emitted_code_unit;
794 }
789795
790796 fn yieldArg(_: *ArgIteratorWindows) bool {
791797 return true;
......@@ -793,6 +799,7 @@ pub const ArgIteratorWindows = struct {
793799 };
794800
795801 fn nextWithStrategy(self: *ArgIteratorWindows, comptime strategy: type) strategy.T {
802 var last_emitted_code_unit: ?u16 = null;
796803 // The first argument (the executable name) uses different parsing rules.
797804 if (self.index == 0) {
798805 if (self.cmd_line.len == 0 or self.cmd_line[0] == 0) {
......@@ -815,15 +822,15 @@ pub const ArgIteratorWindows = struct {
815822 inside_quotes = !inside_quotes;
816823 },
817824 ' ', '\t' => {
818 if (inside_quotes)
819 strategy.emitCharacter(self, char)
820 else {
825 if (inside_quotes) {
826 last_emitted_code_unit = strategy.emitCharacter(self, char, last_emitted_code_unit);
827 } else {
821828 self.index += 1;
822829 return strategy.yieldArg(self);
823830 }
824831 },
825832 else => {
826 strategy.emitCharacter(self, char);
833 last_emitted_code_unit = strategy.emitCharacter(self, char, last_emitted_code_unit);
827834 },
828835 }
829836 }
......@@ -861,29 +868,28 @@ pub const ArgIteratorWindows = struct {
861868 0;
862869 switch (char) {
863870 0 => {
864 strategy.emitBackslashes(self, backslash_count);
871 last_emitted_code_unit = strategy.emitBackslashes(self, backslash_count, last_emitted_code_unit);
865872 return strategy.yieldArg(self);
866873 },
867874 ' ', '\t' => {
868 strategy.emitBackslashes(self, backslash_count);
875 last_emitted_code_unit = strategy.emitBackslashes(self, backslash_count, last_emitted_code_unit);
869876 backslash_count = 0;
870 if (inside_quotes)
871 strategy.emitCharacter(self, char)
872 else
873 return strategy.yieldArg(self);
877 if (inside_quotes) {
878 last_emitted_code_unit = strategy.emitCharacter(self, char, last_emitted_code_unit);
879 } else return strategy.yieldArg(self);
874880 },
875881 '"' => {
876882 const char_is_escaped_quote = backslash_count % 2 != 0;
877 strategy.emitBackslashes(self, backslash_count / 2);
883 last_emitted_code_unit = strategy.emitBackslashes(self, backslash_count / 2, last_emitted_code_unit);
878884 backslash_count = 0;
879885 if (char_is_escaped_quote) {
880 strategy.emitCharacter(self, '"');
886 last_emitted_code_unit = strategy.emitCharacter(self, '"', last_emitted_code_unit);
881887 } else {
882888 if (inside_quotes and
883889 self.index + 1 != self.cmd_line.len and
884890 mem.littleToNative(u16, self.cmd_line[self.index + 1]) == '"')
885891 {
886 strategy.emitCharacter(self, '"');
892 last_emitted_code_unit = strategy.emitCharacter(self, '"', last_emitted_code_unit);
887893 self.index += 1;
888894 } else {
889895 inside_quotes = !inside_quotes;
......@@ -894,9 +900,9 @@ pub const ArgIteratorWindows = struct {
894900 backslash_count += 1;
895901 },
896902 else => {
897 strategy.emitBackslashes(self, backslash_count);
903 last_emitted_code_unit = strategy.emitBackslashes(self, backslash_count, last_emitted_code_unit);
898904 backslash_count = 0;
899 strategy.emitCharacter(self, char);
905 last_emitted_code_unit = strategy.emitCharacter(self, char, last_emitted_code_unit);
900906 },
901907 }
902908 }