authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-09-28 22:23:47+02:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-09-28 22:23:47+02:00
logc27e685e6316764926513f806c0c4a51c53cf7a8
treeafba4c78746499ab9575544d4223fdf12d079edf
parent915c70cb5fb009378d46a199c1beef29559a17a6
parent8e008b39f5ba52b6011f502b1076573a55b2f9be

Merge pull request 'std.zig.stringEscape: Escape invalid/disallowed UTF-8 sequences' (#36923) from squeek502/zig:string-escape-utf8 into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36923

3 files changed, 181 insertions(+), 58 deletions(-)

lib/std/Io/Writer.zig+2-1
......@@ -688,7 +688,8 @@ pub fn writeAll(w: *Writer, bytes: []const u8) Error!void {
688688/// string, everything is passed through unmodified, except for the following
689689/// transformations:
690690/// - escaped: '\n', '\r', '\t', '\\', '"'
691/// - hex-encoded: ASCII control characters
691/// - hex-encoded: ASCII control characters, invalid UTF-8 sequences,
692/// non-ASCII line endings (U+0085, U+2028, U+2029), byte order marks (U+FEFF).
692693/// - "qf": delegates to the `format` method of the type, while double-quote
693694/// escaping.
694695///
lib/std/zig.zig+174-52
......@@ -488,6 +488,8 @@ test fmtId {
488488 try expectFmt("@\"11\\\"23\"", "{f}", .{fmtId("11\"23")});
489489 try expectFmt("@\"11\\x0f23\"", "{f}", .{fmtId("11\x0F23")});
490490
491 try expectFmt("@\"\\r\\\"\\n\\xff😎\\xe2\\x80\\xa8\"", "{f}", .{fmtId("\r\"\n\xFF😎\u{2028}")});
492
491493 // These are technically not currently legal in Zig.
492494 try expectFmt("@\"\"", "{f}", .{fmtId("")});
493495 try expectFmt("@\"\\x00\"", "{f}", .{fmtId("\x00")});
......@@ -541,95 +543,215 @@ test fmtChar {
541543///
542544/// The following transformations are made:
543545/// * escaped: '\n', '\r', '\t', '\\', '"'
544/// * hex-encoded: ascii control characters
546/// * hex-encoded:
547/// * ascii control characters
548/// * invalid UTF-8 sequences
549/// * non-ascii line endings (U+0085, U+2028, U+2029)
550/// * byte order marks (U+FEFF)
545551///
546552/// Everything else is passed through unmodified.
547553pub fn stringEscape(bytes: []const u8, w: *Writer) Writer.Error!void {
548 _ = try stringEscapeCounting(bytes, w);
549}
554 var remaining = bytes.len;
555 while (remaining > 0) {
556 remaining -= try stringEscapeInner(bytes[bytes.len - remaining ..], w);
550557
551pub fn stringEscapeCounting(bytes: []const u8, w: *Writer) Writer.Error!usize {
552 var n: usize = 0;
553 for (bytes) |byte| switch (byte) {
554 '\t' => {
555 try w.writeAll("\\t");
556 n += 2;
557 },
558 '\n' => {
559 try w.writeAll("\\n");
560 n += 2;
561 },
562 '\r' => {
563 try w.writeAll("\\r");
564 n += 2;
565 },
566 '\\' => {
567 try w.writeAll("\\\\");
568 n += 2;
569 },
570 '"' => {
571 try w.writeAll("\\\"");
572 n += 2;
573 },
574 0...8, 11, 12, 14...0x1f, 0x7f => {
558 // Escape the first byte and try again.
559 // Needing to escape the rest is not guaranteed.
560 if (remaining > 0) {
575561 try w.writeAll("\\x");
576 try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' });
577 n += 4;
578 },
579 else => {
580 try w.writeByte(byte);
581 n += 1;
582 },
583 };
584 return n;
562 try w.printInt(bytes[bytes.len - remaining], 16, .lower, .{ .width = 2, .fill = '0' });
563 remaining -= 1;
564 }
565 }
566}
567
568/// Returns the number of bytes consumed from `bytes`, which may be less than `bytes.len`.
569fn stringEscapeInner(bytes: []const u8, w: *Writer) Writer.Error!usize {
570 var i: usize = 0;
571 while (i < bytes.len) {
572 const byte = bytes[i];
573 switch (byte) {
574 '\t' => {
575 try w.writeAll("\\t");
576 },
577 '\n' => {
578 try w.writeAll("\\n");
579 },
580 '\r' => {
581 try w.writeAll("\\r");
582 },
583 '\\' => {
584 try w.writeAll("\\\\");
585 },
586 '"' => {
587 try w.writeAll("\\\"");
588 },
589 0...8, 11, 12, 14...0x1f, 0x7f => {
590 try w.writeAll("\\x");
591 try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' });
592 },
593 0x20, 0x21, 0x23...0x5b, 0x5d...0x7e => {
594 try w.writeByte(byte);
595 },
596 0x80...0xff => {
597 const len, const escape = blk: {
598 const len = std.unicode.utf8ByteSequenceLength(byte) catch break :blk .{ 1, true };
599 if (i + len > bytes.len) {
600 return i;
601 }
602 const sequence = bytes[i..][0..len];
603 const code_point = std.unicode.utf8Decode(sequence) catch |err| switch (err) {
604 error.Utf8CodepointTooLarge => break :blk .{ len, true },
605 else => break :blk .{ 1, true },
606 };
607 switch (code_point) {
608 '\u{feff}', '\u{0085}', '\u{2028}', '\u{2029}' => break :blk .{ len, true },
609 else => break :blk .{ len, false },
610 }
611 };
612
613 const sequence = bytes[i..][0..len];
614 if (escape) {
615 for (sequence) |b| {
616 try w.writeAll("\\x");
617 try w.printInt(b, 16, .lower, .{ .width = 2, .fill = '0' });
618 }
619 } else {
620 try w.writeAll(sequence);
621 }
622 i += len;
623 continue;
624 },
625 }
626 i += 1;
627 }
628 return i;
585629}
586630
587631pub const StringEscapeWriter = struct {
588632 out: *Writer,
589633 writer: Writer,
590634
635 pub const min_buffer_len = 4;
636
591637 pub fn init(out: *Writer, buffer: []u8) @This() {
638 assert(buffer.len >= min_buffer_len);
592639 return .{
593640 .out = out,
594641 .writer = .{
595 .vtable = &.{ .drain = @This().drain },
642 .vtable = &.{ .drain = @This().drain, .flush = @This().flush },
596643 .buffer = buffer,
597644 },
598645 };
599646 }
600647
601648 fn drain(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize {
602 const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w));
603 const out = sew.out;
604 try stringEscape(w.buffered(), out);
605 w.end = 0;
606 var n: usize = 0;
649 var n: usize = try drainBufferRemaining(w, "");
607650 for (data[0 .. data.len - 1]) |bytes| {
608 try stringEscape(bytes, out);
609 n += bytes.len;
651 n += try drainBufferRemaining(w, bytes);
610652 }
611653 const pattern = data[data.len - 1];
612654 for (0..splat) |_| {
613 try stringEscape(pattern, out);
614 n += pattern.len;
655 n += try drainBufferRemaining(w, pattern);
615656 }
616657 return n;
617658 }
659
660 fn flush(w: *Writer) Io.Writer.Error!void {
661 const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w));
662 const out = sew.out;
663 while (w.end != 0) {
664 _ = try drainBufferRemaining(w, "");
665
666 // Escape the first byte and try again.
667 // Needing to escape the rest is not guaranteed.
668 const remaining = w.buffered();
669 if (remaining.len > 0) {
670 try out.writeAll("\\x");
671 try out.printInt(remaining[0], 16, .lower, .{ .width = 2, .fill = '0' });
672 _ = w.consume(1);
673 }
674 }
675 }
676
677 /// Drain from the buffer first, taking from `bytes` as necessary to complete any
678 /// incomplete UTF-8 sequences. Then, consume whatever is remaining of `bytes`,
679 /// storing a possible trailing incomplete UTF-8 sequence in the buffer.
680 ///
681 /// On success, `w.buffered().len` is guaranteed to be < 4.
682 fn drainBufferRemaining(w: *Writer, bytes: []const u8) Io.Writer.Error!usize {
683 const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w));
684 const out = sew.out;
685 var remaining_bytes = bytes;
686 while (w.end != 0) {
687 const n = try stringEscapeInner(w.buffered(), out);
688 _ = w.consume(n);
689
690 if (w.end != 0) {
691 if (remaining_bytes.len == 0) return bytes.len;
692 const len = std.unicode.utf8ByteSequenceLength(w.buffer[0]) catch unreachable;
693 const missing = len - w.end;
694 const available = remaining_bytes.len;
695 const copy_len = @min(missing, available);
696 const start = len - missing;
697 @memcpy(w.buffer[start..][0..copy_len], remaining_bytes[0..copy_len]);
698 w.end = start + copy_len;
699 remaining_bytes = remaining_bytes[copy_len..];
700 }
701 }
702
703 const n = try stringEscapeInner(remaining_bytes, out);
704 if (n < remaining_bytes.len) {
705 const remaining = remaining_bytes.len - n;
706 @memcpy(w.buffer[0..remaining], remaining_bytes[n..]);
707 w.end = remaining;
708 }
709
710 return bytes.len;
711 }
618712};
619713
620test StringEscapeWriter {
621 const bytes = "\x7f\t\n\r\\\"abc";
622 const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc";
714test stringEscape {
715 const bytes = "\x7f\t\n\r\\\"abc\xff\u{feff}\u{0085}\u{2028}\u{2029}\xed\xa0\x80\xf4\x90\x80\x80\xf4\x90a";
716 const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc\\xff\\xef\\xbb\\xbf\\xc2\\x85\\xe2\\x80\\xa8\\xe2\\x80\\xa9\\xed\\xa0\\x80\\xf4\\x90\\x80\\x80\\xf4\\x90a";
623717
624718 var out_buf: [escaped.len]u8 = undefined;
625719 var out: Io.Writer = .fixed(&out_buf);
626 var w: StringEscapeWriter = .init(&out, &.{});
627720
628 const n = try w.writer.write(bytes);
629 try std.testing.expectEqual(bytes.len, n);
721 try stringEscape(bytes, &out);
630722 try std.testing.expectEqualStrings(escaped, out.buffered());
631723}
632724
725test StringEscapeWriter {
726 const bytes = "\x7f\t\n\r\\\"abc\xff\u{feff}\u{0085}\u{2028}\u{2029}\xed\xa0\x80\xf4\x90\x80\x80\xf4\x90a";
727 const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc\\xff\\xef\\xbb\\xbf\\xc2\\x85\\xe2\\x80\\xa8\\xe2\\x80\\xa9\\xed\\xa0\\x80\\xf4\\x90\\x80\\x80\\xf4\\x90a";
728
729 var sew_buf: [StringEscapeWriter.min_buffer_len]u8 = undefined;
730 {
731 var out_buf: [escaped.len]u8 = undefined;
732 var out: Io.Writer = .fixed(&out_buf);
733 var w: StringEscapeWriter = .init(&out, &sew_buf);
734
735 const n = try w.writer.write(bytes);
736 try w.writer.flush();
737
738 try std.testing.expectEqual(bytes.len, n);
739 try std.testing.expectEqualStrings(escaped, out.buffered());
740 }
741 {
742 var out_buf: [escaped.len]u8 = undefined;
743 var out: Io.Writer = .fixed(&out_buf);
744 var w: StringEscapeWriter = .init(&out, &sew_buf);
745
746 for (bytes) |byte| {
747 try w.writer.writeByte(byte);
748 }
749 try w.writer.flush();
750
751 try std.testing.expectEqualStrings(escaped, out.buffered());
752 }
753}
754
633755/// Print as escaped contents of a single-quoted string.
634756pub fn charEscape(codepoint: u21, w: *Writer) Writer.Error!void {
635757 switch (codepoint) {
lib/std/zig/parser_generated_oracle.zig+5-5
......@@ -3073,7 +3073,7 @@ const Parser = struct {
30733073 return blk_0: {
30743074 const pos_0 = p.i;
30753075 if (blk_1: {
3076 if (std.mem.startsWith(u8, p.source[p.i..], "")) {
3076 if (std.mem.startsWith(u8, p.source[p.i..], "\xef\xbb\xbf")) {
30773077 p.i += 3;
30783078 break :blk_1 true;
30793079 }
......@@ -3145,7 +3145,7 @@ const Parser = struct {
31453145 return blk_0: {
31463146 const pos_0 = p.i;
31473147 if (blk_1: {
3148 if (std.mem.startsWith(u8, p.source[p.i..], "�")) {
3148 if (std.mem.startsWith(u8, p.source[p.i..], "\xf4")) {
31493149 p.i += 1;
31503150 break :blk_1 true;
31513151 }
......@@ -3201,7 +3201,7 @@ const Parser = struct {
32013201 return blk_0: {
32023202 const pos_0 = p.i;
32033203 if (blk_1: {
3204 if (std.mem.startsWith(u8, p.source[p.i..], "�")) {
3204 if (std.mem.startsWith(u8, p.source[p.i..], "\xf0")) {
32053205 p.i += 1;
32063206 break :blk_1 true;
32073207 }
......@@ -3257,7 +3257,7 @@ const Parser = struct {
32573257 return blk_0: {
32583258 const pos_0 = p.i;
32593259 if (blk_1: {
3260 if (std.mem.startsWith(u8, p.source[p.i..], "�")) {
3260 if (std.mem.startsWith(u8, p.source[p.i..], "\xed")) {
32613261 p.i += 1;
32623262 break :blk_1 true;
32633263 }
......@@ -3313,7 +3313,7 @@ const Parser = struct {
33133313 return blk_0: {
33143314 const pos_0 = p.i;
33153315 if (blk_1: {
3316 if (std.mem.startsWith(u8, p.source[p.i..], "�")) {
3316 if (std.mem.startsWith(u8, p.source[p.i..], "\xe0")) {
33173317 p.i += 1;
33183318 break :blk_1 true;
33193319 }