authorgravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-09-28 22:23:47+02:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-09-28 22:23:47+02:00
logc27e685e6316764926513f806c0c4a51c53cf7a8
treeafba4c78746499ab9575544d4223fdf12d079edf
parent915c70cb5fb009378d46a199c1beef29559a17a6
parent8e008b39f5ba52b6011f502b1076573a55b2f9be

Merge pull request 'std.zig.stringEscape: Escape invalid/disallowed UTF-8 sequences' (#36923) from squeek502/zig:string-escape-utf8 into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36923

3 files changed, 181 insertions(+), 58 deletions(-)

lib/std/Io/Writer.zig+2-1
...@@ -688,7 +688,8 @@ pub fn writeAll(w: *Writer, bytes: []const u8) Error!void {...@@ -688,7 +688,8 @@ pub fn writeAll(w: *Writer, bytes: []const u8) Error!void {
688/// string, everything is passed through unmodified, except for the following688/// string, everything is passed through unmodified, except for the following
689/// transformations:689/// transformations:
690/// - escaped: '\n', '\r', '\t', '\\', '"'690/// - escaped: '\n', '\r', '\t', '\\', '"'
691/// - hex-encoded: ASCII control characters691/// - hex-encoded: ASCII control characters, invalid UTF-8 sequences,
692/// non-ASCII line endings (U+0085, U+2028, U+2029), byte order marks (U+FEFF).
692/// - "qf": delegates to the `format` method of the type, while double-quote693/// - "qf": delegates to the `format` method of the type, while double-quote
693/// escaping.694/// escaping.
694///695///
lib/std/zig.zig+174-52
...@@ -488,6 +488,8 @@ test fmtId {...@@ -488,6 +488,8 @@ test fmtId {
488 try expectFmt("@\"11\\\"23\"", "{f}", .{fmtId("11\"23")});488 try expectFmt("@\"11\\\"23\"", "{f}", .{fmtId("11\"23")});
489 try expectFmt("@\"11\\x0f23\"", "{f}", .{fmtId("11\x0F23")});489 try expectFmt("@\"11\\x0f23\"", "{f}", .{fmtId("11\x0F23")});
490490
491 try expectFmt("@\"\\r\\\"\\n\\xff😎\\xe2\\x80\\xa8\"", "{f}", .{fmtId("\r\"\n\xFF😎\u{2028}")});
492
491 // These are technically not currently legal in Zig.493 // These are technically not currently legal in Zig.
492 try expectFmt("@\"\"", "{f}", .{fmtId("")});494 try expectFmt("@\"\"", "{f}", .{fmtId("")});
493 try expectFmt("@\"\\x00\"", "{f}", .{fmtId("\x00")});495 try expectFmt("@\"\\x00\"", "{f}", .{fmtId("\x00")});
...@@ -541,95 +543,215 @@ test fmtChar {...@@ -541,95 +543,215 @@ test fmtChar {
541///543///
542/// The following transformations are made:544/// The following transformations are made:
543/// * escaped: '\n', '\r', '\t', '\\', '"'545/// * escaped: '\n', '\r', '\t', '\\', '"'
544/// * hex-encoded: ascii control characters546/// * hex-encoded:
547/// * ascii control characters
548/// * invalid UTF-8 sequences
549/// * non-ascii line endings (U+0085, U+2028, U+2029)
550/// * byte order marks (U+FEFF)
545///551///
546/// Everything else is passed through unmodified.552/// Everything else is passed through unmodified.
547pub fn stringEscape(bytes: []const u8, w: *Writer) Writer.Error!void {553pub fn stringEscape(bytes: []const u8, w: *Writer) Writer.Error!void {
548 _ = try stringEscapeCounting(bytes, w);554 var remaining = bytes.len;
549}555 while (remaining > 0) {
556 remaining -= try stringEscapeInner(bytes[bytes.len - remaining ..], w);
550557
551pub fn stringEscapeCounting(bytes: []const u8, w: *Writer) Writer.Error!usize {558 // Escape the first byte and try again.
552 var n: usize = 0;559 // Needing to escape the rest is not guaranteed.
553 for (bytes) |byte| switch (byte) {560 if (remaining > 0) {
554 '\t' => {
555 try w.writeAll("\\t");
556 n += 2;
557 },
558 '\n' => {
559 try w.writeAll("\\n");
560 n += 2;
561 },
562 '\r' => {
563 try w.writeAll("\\r");
564 n += 2;
565 },
566 '\\' => {
567 try w.writeAll("\\\\");
568 n += 2;
569 },
570 '"' => {
571 try w.writeAll("\\\"");
572 n += 2;
573 },
574 0...8, 11, 12, 14...0x1f, 0x7f => {
575 try w.writeAll("\\x");561 try w.writeAll("\\x");
576 try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' });562 try w.printInt(bytes[bytes.len - remaining], 16, .lower, .{ .width = 2, .fill = '0' });
577 n += 4;563 remaining -= 1;
578 },564 }
579 else => {565 }
580 try w.writeByte(byte);566}
581 n += 1;567
582 },568/// Returns the number of bytes consumed from `bytes`, which may be less than `bytes.len`.
583 };569fn stringEscapeInner(bytes: []const u8, w: *Writer) Writer.Error!usize {
584 return n;570 var i: usize = 0;
571 while (i < bytes.len) {
572 const byte = bytes[i];
573 switch (byte) {
574 '\t' => {
575 try w.writeAll("\\t");
576 },
577 '\n' => {
578 try w.writeAll("\\n");
579 },
580 '\r' => {
581 try w.writeAll("\\r");
582 },
583 '\\' => {
584 try w.writeAll("\\\\");
585 },
586 '"' => {
587 try w.writeAll("\\\"");
588 },
589 0...8, 11, 12, 14...0x1f, 0x7f => {
590 try w.writeAll("\\x");
591 try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' });
592 },
593 0x20, 0x21, 0x23...0x5b, 0x5d...0x7e => {
594 try w.writeByte(byte);
595 },
596 0x80...0xff => {
597 const len, const escape = blk: {
598 const len = std.unicode.utf8ByteSequenceLength(byte) catch break :blk .{ 1, true };
599 if (i + len > bytes.len) {
600 return i;
601 }
602 const sequence = bytes[i..][0..len];
603 const code_point = std.unicode.utf8Decode(sequence) catch |err| switch (err) {
604 error.Utf8CodepointTooLarge => break :blk .{ len, true },
605 else => break :blk .{ 1, true },
606 };
607 switch (code_point) {
608 '\u{feff}', '\u{0085}', '\u{2028}', '\u{2029}' => break :blk .{ len, true },
609 else => break :blk .{ len, false },
610 }
611 };
612
613 const sequence = bytes[i..][0..len];
614 if (escape) {
615 for (sequence) |b| {
616 try w.writeAll("\\x");
617 try w.printInt(b, 16, .lower, .{ .width = 2, .fill = '0' });
618 }
619 } else {
620 try w.writeAll(sequence);
621 }
622 i += len;
623 continue;
624 },
625 }
626 i += 1;
627 }
628 return i;
585}629}
586630
587pub const StringEscapeWriter = struct {631pub const StringEscapeWriter = struct {
588 out: *Writer,632 out: *Writer,
589 writer: Writer,633 writer: Writer,
590634
635 pub const min_buffer_len = 4;
636
591 pub fn init(out: *Writer, buffer: []u8) @This() {637 pub fn init(out: *Writer, buffer: []u8) @This() {
638 assert(buffer.len >= min_buffer_len);
592 return .{639 return .{
593 .out = out,640 .out = out,
594 .writer = .{641 .writer = .{
595 .vtable = &.{ .drain = @This().drain },642 .vtable = &.{ .drain = @This().drain, .flush = @This().flush },
596 .buffer = buffer,643 .buffer = buffer,
597 },644 },
598 };645 };
599 }646 }
600647
601 fn drain(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize {648 fn drain(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize {
602 const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w));649 var n: usize = try drainBufferRemaining(w, "");
603 const out = sew.out;
604 try stringEscape(w.buffered(), out);
605 w.end = 0;
606 var n: usize = 0;
607 for (data[0 .. data.len - 1]) |bytes| {650 for (data[0 .. data.len - 1]) |bytes| {
608 try stringEscape(bytes, out);651 n += try drainBufferRemaining(w, bytes);
609 n += bytes.len;
610 }652 }
611 const pattern = data[data.len - 1];653 const pattern = data[data.len - 1];
612 for (0..splat) |_| {654 for (0..splat) |_| {
613 try stringEscape(pattern, out);655 n += try drainBufferRemaining(w, pattern);
614 n += pattern.len;
615 }656 }
616 return n;657 return n;
617 }658 }
659
660 fn flush(w: *Writer) Io.Writer.Error!void {
661 const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w));
662 const out = sew.out;
663 while (w.end != 0) {
664 _ = try drainBufferRemaining(w, "");
665
666 // Escape the first byte and try again.
667 // Needing to escape the rest is not guaranteed.
668 const remaining = w.buffered();
669 if (remaining.len > 0) {
670 try out.writeAll("\\x");
671 try out.printInt(remaining[0], 16, .lower, .{ .width = 2, .fill = '0' });
672 _ = w.consume(1);
673 }
674 }
675 }
676
677 /// Drain from the buffer first, taking from `bytes` as necessary to complete any
678 /// incomplete UTF-8 sequences. Then, consume whatever is remaining of `bytes`,
679 /// storing a possible trailing incomplete UTF-8 sequence in the buffer.
680 ///
681 /// On success, `w.buffered().len` is guaranteed to be < 4.
682 fn drainBufferRemaining(w: *Writer, bytes: []const u8) Io.Writer.Error!usize {
683 const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w));
684 const out = sew.out;
685 var remaining_bytes = bytes;
686 while (w.end != 0) {
687 const n = try stringEscapeInner(w.buffered(), out);
688 _ = w.consume(n);
689
690 if (w.end != 0) {
691 if (remaining_bytes.len == 0) return bytes.len;
692 const len = std.unicode.utf8ByteSequenceLength(w.buffer[0]) catch unreachable;
693 const missing = len - w.end;
694 const available = remaining_bytes.len;
695 const copy_len = @min(missing, available);
696 const start = len - missing;
697 @memcpy(w.buffer[start..][0..copy_len], remaining_bytes[0..copy_len]);
698 w.end = start + copy_len;
699 remaining_bytes = remaining_bytes[copy_len..];
700 }
701 }
702
703 const n = try stringEscapeInner(remaining_bytes, out);
704 if (n < remaining_bytes.len) {
705 const remaining = remaining_bytes.len - n;
706 @memcpy(w.buffer[0..remaining], remaining_bytes[n..]);
707 w.end = remaining;
708 }
709
710 return bytes.len;
711 }
618};712};
619713
620test StringEscapeWriter {714test stringEscape {
621 const bytes = "\x7f\t\n\r\\\"abc";715 const bytes = "\x7f\t\n\r\\\"abc\xff\u{feff}\u{0085}\u{2028}\u{2029}\xed\xa0\x80\xf4\x90\x80\x80\xf4\x90a";
622 const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc";716 const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc\\xff\\xef\\xbb\\xbf\\xc2\\x85\\xe2\\x80\\xa8\\xe2\\x80\\xa9\\xed\\xa0\\x80\\xf4\\x90\\x80\\x80\\xf4\\x90a";
623717
624 var out_buf: [escaped.len]u8 = undefined;718 var out_buf: [escaped.len]u8 = undefined;
625 var out: Io.Writer = .fixed(&out_buf);719 var out: Io.Writer = .fixed(&out_buf);
626 var w: StringEscapeWriter = .init(&out, &.{});
627720
628 const n = try w.writer.write(bytes);721 try stringEscape(bytes, &out);
629 try std.testing.expectEqual(bytes.len, n);
630 try std.testing.expectEqualStrings(escaped, out.buffered());722 try std.testing.expectEqualStrings(escaped, out.buffered());
631}723}
632724
725test StringEscapeWriter {
726 const bytes = "\x7f\t\n\r\\\"abc\xff\u{feff}\u{0085}\u{2028}\u{2029}\xed\xa0\x80\xf4\x90\x80\x80\xf4\x90a";
727 const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc\\xff\\xef\\xbb\\xbf\\xc2\\x85\\xe2\\x80\\xa8\\xe2\\x80\\xa9\\xed\\xa0\\x80\\xf4\\x90\\x80\\x80\\xf4\\x90a";
728
729 var sew_buf: [StringEscapeWriter.min_buffer_len]u8 = undefined;
730 {
731 var out_buf: [escaped.len]u8 = undefined;
732 var out: Io.Writer = .fixed(&out_buf);
733 var w: StringEscapeWriter = .init(&out, &sew_buf);
734
735 const n = try w.writer.write(bytes);
736 try w.writer.flush();
737
738 try std.testing.expectEqual(bytes.len, n);
739 try std.testing.expectEqualStrings(escaped, out.buffered());
740 }
741 {
742 var out_buf: [escaped.len]u8 = undefined;
743 var out: Io.Writer = .fixed(&out_buf);
744 var w: StringEscapeWriter = .init(&out, &sew_buf);
745
746 for (bytes) |byte| {
747 try w.writer.writeByte(byte);
748 }
749 try w.writer.flush();
750
751 try std.testing.expectEqualStrings(escaped, out.buffered());
752 }
753}
754
633/// Print as escaped contents of a single-quoted string.755/// Print as escaped contents of a single-quoted string.
634pub fn charEscape(codepoint: u21, w: *Writer) Writer.Error!void {756pub fn charEscape(codepoint: u21, w: *Writer) Writer.Error!void {
635 switch (codepoint) {757 switch (codepoint) {
lib/std/zig/parser_generated_oracle.zig+5-5
...@@ -3073,7 +3073,7 @@ const Parser = struct {...@@ -3073,7 +3073,7 @@ const Parser = struct {
3073 return blk_0: {3073 return blk_0: {
3074 const pos_0 = p.i;3074 const pos_0 = p.i;
3075 if (blk_1: {3075 if (blk_1: {
3076 if (std.mem.startsWith(u8, p.source[p.i..], "")) {3076 if (std.mem.startsWith(u8, p.source[p.i..], "\xef\xbb\xbf")) {
3077 p.i += 3;3077 p.i += 3;
3078 break :blk_1 true;3078 break :blk_1 true;
3079 }3079 }
...@@ -3145,7 +3145,7 @@ const Parser = struct {...@@ -3145,7 +3145,7 @@ const Parser = struct {
3145 return blk_0: {3145 return blk_0: {
3146 const pos_0 = p.i;3146 const pos_0 = p.i;
3147 if (blk_1: {3147 if (blk_1: {
3148 if (std.mem.startsWith(u8, p.source[p.i..], "�")) {3148 if (std.mem.startsWith(u8, p.source[p.i..], "\xf4")) {
3149 p.i += 1;3149 p.i += 1;
3150 break :blk_1 true;3150 break :blk_1 true;
3151 }3151 }
...@@ -3201,7 +3201,7 @@ const Parser = struct {...@@ -3201,7 +3201,7 @@ const Parser = struct {
3201 return blk_0: {3201 return blk_0: {
3202 const pos_0 = p.i;3202 const pos_0 = p.i;
3203 if (blk_1: {3203 if (blk_1: {
3204 if (std.mem.startsWith(u8, p.source[p.i..], "�")) {3204 if (std.mem.startsWith(u8, p.source[p.i..], "\xf0")) {
3205 p.i += 1;3205 p.i += 1;
3206 break :blk_1 true;3206 break :blk_1 true;
3207 }3207 }
...@@ -3257,7 +3257,7 @@ const Parser = struct {...@@ -3257,7 +3257,7 @@ const Parser = struct {
3257 return blk_0: {3257 return blk_0: {
3258 const pos_0 = p.i;3258 const pos_0 = p.i;
3259 if (blk_1: {3259 if (blk_1: {
3260 if (std.mem.startsWith(u8, p.source[p.i..], "�")) {3260 if (std.mem.startsWith(u8, p.source[p.i..], "\xed")) {
3261 p.i += 1;3261 p.i += 1;
3262 break :blk_1 true;3262 break :blk_1 true;
3263 }3263 }
...@@ -3313,7 +3313,7 @@ const Parser = struct {...@@ -3313,7 +3313,7 @@ const Parser = struct {
3313 return blk_0: {3313 return blk_0: {
3314 const pos_0 = p.i;3314 const pos_0 = p.i;
3315 if (blk_1: {3315 if (blk_1: {
3316 if (std.mem.startsWith(u8, p.source[p.i..], "�")) {3316 if (std.mem.startsWith(u8, p.source[p.i..], "\xe0")) {
3317 p.i += 1;3317 p.i += 1;
3318 break :blk_1 true;3318 break :blk_1 true;
3319 }3319 }