| ... | @@ -488,6 +488,8 @@ test fmtId { | ... | @@ -488,6 +488,8 @@ test fmtId { |
| 488 | try expectFmt("@\"11\\\"23\"", "{f}", .{fmtId("11\"23")}); | 488 | try expectFmt("@\"11\\\"23\"", "{f}", .{fmtId("11\"23")}); |
| 489 | try expectFmt("@\"11\\x0f23\"", "{f}", .{fmtId("11\x0F23")}); | 489 | try expectFmt("@\"11\\x0f23\"", "{f}", .{fmtId("11\x0F23")}); |
| 490 | | 490 | |
| | 491 | try expectFmt("@\"\\r\\\"\\n\\xff😎\\xe2\\x80\\xa8\"", "{f}", .{fmtId("\r\"\n\xFF😎\u{2028}")}); |
| | 492 | |
| 491 | // These are technically not currently legal in Zig. | 493 | // These are technically not currently legal in Zig. |
| 492 | try expectFmt("@\"\"", "{f}", .{fmtId("")}); | 494 | try expectFmt("@\"\"", "{f}", .{fmtId("")}); |
| 493 | try expectFmt("@\"\\x00\"", "{f}", .{fmtId("\x00")}); | 495 | try expectFmt("@\"\\x00\"", "{f}", .{fmtId("\x00")}); |
| ... | @@ -541,95 +543,215 @@ test fmtChar { | ... | @@ -541,95 +543,215 @@ test fmtChar { |
| 541 | /// | 543 | /// |
| 542 | /// The following transformations are made: | 544 | /// The following transformations are made: |
| 543 | /// * escaped: '\n', '\r', '\t', '\\', '"' | 545 | /// * escaped: '\n', '\r', '\t', '\\', '"' |
| 544 | /// * hex-encoded: ascii control characters | 546 | /// * hex-encoded: |
| | 547 | /// * ascii control characters |
| | 548 | /// * invalid UTF-8 sequences |
| | 549 | /// * non-ascii line endings (U+0085, U+2028, U+2029) |
| | 550 | /// * byte order marks (U+FEFF) |
| 545 | /// | 551 | /// |
| 546 | /// Everything else is passed through unmodified. | 552 | /// Everything else is passed through unmodified. |
| 547 | pub fn stringEscape(bytes: []const u8, w: *Writer) Writer.Error!void { | 553 | pub fn stringEscape(bytes: []const u8, w: *Writer) Writer.Error!void { |
| 548 | _ = try stringEscapeCounting(bytes, w); | 554 | var remaining = bytes.len; |
| 549 | } | 555 | while (remaining > 0) { |
| | 556 | remaining -= try stringEscapeInner(bytes[bytes.len - remaining ..], w); |
| 550 | | 557 | |
| 551 | pub fn stringEscapeCounting(bytes: []const u8, w: *Writer) Writer.Error!usize { | 558 | // Escape the first byte and try again. |
| 552 | var n: usize = 0; | 559 | // Needing to escape the rest is not guaranteed. |
| 553 | for (bytes) |byte| switch (byte) { | 560 | if (remaining > 0) { |
| 554 | '\t' => { | | |
| 555 | try w.writeAll("\\t"); | | |
| 556 | n += 2; | | |
| 557 | }, | | |
| 558 | '\n' => { | | |
| 559 | try w.writeAll("\\n"); | | |
| 560 | n += 2; | | |
| 561 | }, | | |
| 562 | '\r' => { | | |
| 563 | try w.writeAll("\\r"); | | |
| 564 | n += 2; | | |
| 565 | }, | | |
| 566 | '\\' => { | | |
| 567 | try w.writeAll("\\\\"); | | |
| 568 | n += 2; | | |
| 569 | }, | | |
| 570 | '"' => { | | |
| 571 | try w.writeAll("\\\""); | | |
| 572 | n += 2; | | |
| 573 | }, | | |
| 574 | 0...8, 11, 12, 14...0x1f, 0x7f => { | | |
| 575 | try w.writeAll("\\x"); | 561 | try w.writeAll("\\x"); |
| 576 | try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' }); | 562 | try w.printInt(bytes[bytes.len - remaining], 16, .lower, .{ .width = 2, .fill = '0' }); |
| 577 | n += 4; | 563 | remaining -= 1; |
| 578 | }, | 564 | } |
| 579 | else => { | 565 | } |
| 580 | try w.writeByte(byte); | 566 | } |
| 581 | n += 1; | 567 | |
| 582 | }, | 568 | /// Returns the number of bytes consumed from `bytes`, which may be less than `bytes.len`. |
| 583 | }; | 569 | fn stringEscapeInner(bytes: []const u8, w: *Writer) Writer.Error!usize { |
| 584 | return n; | 570 | var i: usize = 0; |
| | 571 | while (i < bytes.len) { |
| | 572 | const byte = bytes[i]; |
| | 573 | switch (byte) { |
| | 574 | '\t' => { |
| | 575 | try w.writeAll("\\t"); |
| | 576 | }, |
| | 577 | '\n' => { |
| | 578 | try w.writeAll("\\n"); |
| | 579 | }, |
| | 580 | '\r' => { |
| | 581 | try w.writeAll("\\r"); |
| | 582 | }, |
| | 583 | '\\' => { |
| | 584 | try w.writeAll("\\\\"); |
| | 585 | }, |
| | 586 | '"' => { |
| | 587 | try w.writeAll("\\\""); |
| | 588 | }, |
| | 589 | 0...8, 11, 12, 14...0x1f, 0x7f => { |
| | 590 | try w.writeAll("\\x"); |
| | 591 | try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' }); |
| | 592 | }, |
| | 593 | 0x20, 0x21, 0x23...0x5b, 0x5d...0x7e => { |
| | 594 | try w.writeByte(byte); |
| | 595 | }, |
| | 596 | 0x80...0xff => { |
| | 597 | const len, const escape = blk: { |
| | 598 | const len = std.unicode.utf8ByteSequenceLength(byte) catch break :blk .{ 1, true }; |
| | 599 | if (i + len > bytes.len) { |
| | 600 | return i; |
| | 601 | } |
| | 602 | const sequence = bytes[i..][0..len]; |
| | 603 | const code_point = std.unicode.utf8Decode(sequence) catch |err| switch (err) { |
| | 604 | error.Utf8CodepointTooLarge => break :blk .{ len, true }, |
| | 605 | else => break :blk .{ 1, true }, |
| | 606 | }; |
| | 607 | switch (code_point) { |
| | 608 | '\u{feff}', '\u{0085}', '\u{2028}', '\u{2029}' => break :blk .{ len, true }, |
| | 609 | else => break :blk .{ len, false }, |
| | 610 | } |
| | 611 | }; |
| | 612 | |
| | 613 | const sequence = bytes[i..][0..len]; |
| | 614 | if (escape) { |
| | 615 | for (sequence) |b| { |
| | 616 | try w.writeAll("\\x"); |
| | 617 | try w.printInt(b, 16, .lower, .{ .width = 2, .fill = '0' }); |
| | 618 | } |
| | 619 | } else { |
| | 620 | try w.writeAll(sequence); |
| | 621 | } |
| | 622 | i += len; |
| | 623 | continue; |
| | 624 | }, |
| | 625 | } |
| | 626 | i += 1; |
| | 627 | } |
| | 628 | return i; |
| 585 | } | 629 | } |
| 586 | | 630 | |
| 587 | pub const StringEscapeWriter = struct { | 631 | pub const StringEscapeWriter = struct { |
| 588 | out: *Writer, | 632 | out: *Writer, |
| 589 | writer: Writer, | 633 | writer: Writer, |
| 590 | | 634 | |
| | 635 | pub const min_buffer_len = 4; |
| | 636 | |
| 591 | pub fn init(out: *Writer, buffer: []u8) @This() { | 637 | pub fn init(out: *Writer, buffer: []u8) @This() { |
| | 638 | assert(buffer.len >= min_buffer_len); |
| 592 | return .{ | 639 | return .{ |
| 593 | .out = out, | 640 | .out = out, |
| 594 | .writer = .{ | 641 | .writer = .{ |
| 595 | .vtable = &.{ .drain = @This().drain }, | 642 | .vtable = &.{ .drain = @This().drain, .flush = @This().flush }, |
| 596 | .buffer = buffer, | 643 | .buffer = buffer, |
| 597 | }, | 644 | }, |
| 598 | }; | 645 | }; |
| 599 | } | 646 | } |
| 600 | | 647 | |
| 601 | fn drain(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize { | 648 | fn drain(w: *Writer, data: []const []const u8, splat: usize) Io.Writer.Error!usize { |
| 602 | const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w)); | 649 | var n: usize = try drainBufferRemaining(w, ""); |
| 603 | const out = sew.out; | | |
| 604 | try stringEscape(w.buffered(), out); | | |
| 605 | w.end = 0; | | |
| 606 | var n: usize = 0; | | |
| 607 | for (data[0 .. data.len - 1]) |bytes| { | 650 | for (data[0 .. data.len - 1]) |bytes| { |
| 608 | try stringEscape(bytes, out); | 651 | n += try drainBufferRemaining(w, bytes); |
| 609 | n += bytes.len; | | |
| 610 | } | 652 | } |
| 611 | const pattern = data[data.len - 1]; | 653 | const pattern = data[data.len - 1]; |
| 612 | for (0..splat) |_| { | 654 | for (0..splat) |_| { |
| 613 | try stringEscape(pattern, out); | 655 | n += try drainBufferRemaining(w, pattern); |
| 614 | n += pattern.len; | | |
| 615 | } | 656 | } |
| 616 | return n; | 657 | return n; |
| 617 | } | 658 | } |
| | 659 | |
| | 660 | fn flush(w: *Writer) Io.Writer.Error!void { |
| | 661 | const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w)); |
| | 662 | const out = sew.out; |
| | 663 | while (w.end != 0) { |
| | 664 | _ = try drainBufferRemaining(w, ""); |
| | 665 | |
| | 666 | // Escape the first byte and try again. |
| | 667 | // Needing to escape the rest is not guaranteed. |
| | 668 | const remaining = w.buffered(); |
| | 669 | if (remaining.len > 0) { |
| | 670 | try out.writeAll("\\x"); |
| | 671 | try out.printInt(remaining[0], 16, .lower, .{ .width = 2, .fill = '0' }); |
| | 672 | _ = w.consume(1); |
| | 673 | } |
| | 674 | } |
| | 675 | } |
| | 676 | |
| | 677 | /// Drain from the buffer first, taking from `bytes` as necessary to complete any |
| | 678 | /// incomplete UTF-8 sequences. Then, consume whatever is remaining of `bytes`, |
| | 679 | /// storing a possible trailing incomplete UTF-8 sequence in the buffer. |
| | 680 | /// |
| | 681 | /// On success, `w.buffered().len` is guaranteed to be < 4. |
| | 682 | fn drainBufferRemaining(w: *Writer, bytes: []const u8) Io.Writer.Error!usize { |
| | 683 | const sew: *StringEscapeWriter = @alignCast(@fieldParentPtr("writer", w)); |
| | 684 | const out = sew.out; |
| | 685 | var remaining_bytes = bytes; |
| | 686 | while (w.end != 0) { |
| | 687 | const n = try stringEscapeInner(w.buffered(), out); |
| | 688 | _ = w.consume(n); |
| | 689 | |
| | 690 | if (w.end != 0) { |
| | 691 | if (remaining_bytes.len == 0) return bytes.len; |
| | 692 | const len = std.unicode.utf8ByteSequenceLength(w.buffer[0]) catch unreachable; |
| | 693 | const missing = len - w.end; |
| | 694 | const available = remaining_bytes.len; |
| | 695 | const copy_len = @min(missing, available); |
| | 696 | const start = len - missing; |
| | 697 | @memcpy(w.buffer[start..][0..copy_len], remaining_bytes[0..copy_len]); |
| | 698 | w.end = start + copy_len; |
| | 699 | remaining_bytes = remaining_bytes[copy_len..]; |
| | 700 | } |
| | 701 | } |
| | 702 | |
| | 703 | const n = try stringEscapeInner(remaining_bytes, out); |
| | 704 | if (n < remaining_bytes.len) { |
| | 705 | const remaining = remaining_bytes.len - n; |
| | 706 | @memcpy(w.buffer[0..remaining], remaining_bytes[n..]); |
| | 707 | w.end = remaining; |
| | 708 | } |
| | 709 | |
| | 710 | return bytes.len; |
| | 711 | } |
| 618 | }; | 712 | }; |
| 619 | | 713 | |
| 620 | test StringEscapeWriter { | 714 | test stringEscape { |
| 621 | const bytes = "\x7f\t\n\r\\\"abc"; | 715 | const bytes = "\x7f\t\n\r\\\"abc\xff\u{feff}\u{0085}\u{2028}\u{2029}\xed\xa0\x80\xf4\x90\x80\x80\xf4\x90a"; |
| 622 | const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc"; | 716 | const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc\\xff\\xef\\xbb\\xbf\\xc2\\x85\\xe2\\x80\\xa8\\xe2\\x80\\xa9\\xed\\xa0\\x80\\xf4\\x90\\x80\\x80\\xf4\\x90a"; |
| 623 | | 717 | |
| 624 | var out_buf: [escaped.len]u8 = undefined; | 718 | var out_buf: [escaped.len]u8 = undefined; |
| 625 | var out: Io.Writer = .fixed(&out_buf); | 719 | var out: Io.Writer = .fixed(&out_buf); |
| 626 | var w: StringEscapeWriter = .init(&out, &.{}); | | |
| 627 | | 720 | |
| 628 | const n = try w.writer.write(bytes); | 721 | try stringEscape(bytes, &out); |
| 629 | try std.testing.expectEqual(bytes.len, n); | | |
| 630 | try std.testing.expectEqualStrings(escaped, out.buffered()); | 722 | try std.testing.expectEqualStrings(escaped, out.buffered()); |
| 631 | } | 723 | } |
| 632 | | 724 | |
| | 725 | test StringEscapeWriter { |
| | 726 | const bytes = "\x7f\t\n\r\\\"abc\xff\u{feff}\u{0085}\u{2028}\u{2029}\xed\xa0\x80\xf4\x90\x80\x80\xf4\x90a"; |
| | 727 | const escaped = "\\x7f\\t\\n\\r\\\\\\\"abc\\xff\\xef\\xbb\\xbf\\xc2\\x85\\xe2\\x80\\xa8\\xe2\\x80\\xa9\\xed\\xa0\\x80\\xf4\\x90\\x80\\x80\\xf4\\x90a"; |
| | 728 | |
| | 729 | var sew_buf: [StringEscapeWriter.min_buffer_len]u8 = undefined; |
| | 730 | { |
| | 731 | var out_buf: [escaped.len]u8 = undefined; |
| | 732 | var out: Io.Writer = .fixed(&out_buf); |
| | 733 | var w: StringEscapeWriter = .init(&out, &sew_buf); |
| | 734 | |
| | 735 | const n = try w.writer.write(bytes); |
| | 736 | try w.writer.flush(); |
| | 737 | |
| | 738 | try std.testing.expectEqual(bytes.len, n); |
| | 739 | try std.testing.expectEqualStrings(escaped, out.buffered()); |
| | 740 | } |
| | 741 | { |
| | 742 | var out_buf: [escaped.len]u8 = undefined; |
| | 743 | var out: Io.Writer = .fixed(&out_buf); |
| | 744 | var w: StringEscapeWriter = .init(&out, &sew_buf); |
| | 745 | |
| | 746 | for (bytes) |byte| { |
| | 747 | try w.writer.writeByte(byte); |
| | 748 | } |
| | 749 | try w.writer.flush(); |
| | 750 | |
| | 751 | try std.testing.expectEqualStrings(escaped, out.buffered()); |
| | 752 | } |
| | 753 | } |
| | 754 | |
| 633 | /// Print as escaped contents of a single-quoted string. | 755 | /// Print as escaped contents of a single-quoted string. |
| 634 | pub fn charEscape(codepoint: u21, w: *Writer) Writer.Error!void { | 756 | pub fn charEscape(codepoint: u21, w: *Writer) Writer.Error!void { |
| 635 | switch (codepoint) { | 757 | switch (codepoint) { |
Ryan Liptak <squeek502@hotmail.com> 2026-09-08 02:36:05-07:00