authorgravatar for ben@benburkert.comBen Burkert <ben@benburkert.com> 2026-09-30 04:22:13+02:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-09-30 04:22:13+02:00
log087ee6f4efce2bb635c11282a43b0014e56f366f
tree603fba260af531be36e9a38e6a9cdc67b2609154
parentbbb48690de0d320e3a3d6436d28acfb8029f80c5

zon: fix serialization of invalid UTF-8 (#36754)

The `std.zon.Serializer.string` method would previously panic with `attempt to unwrap error: Utf8InvalidStartByte` when invoked with a string parameter containing invalid UTF-8 while the `escape_non_ascii` option was enabled, and also allow invalid UTF-8 to pass through unescaped when `escape_non_ascii` was disabled. This commit makes invalid UTF-8 get escaped into hex sequences (e.g. "\xff") regardless of the `escape_non_ascii` setting. Related: #36923 Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36754 Reviewed-by: Ryan Liptak <squeek502@noreply.codeberg.org>

1 files changed, 75 insertions(+), 34 deletions(-)

lib/std/zon/Serializer.zig+75-34
......@@ -410,51 +410,92 @@ pub const StringOptions = struct {
410410
411411/// Like `value`, but always serializes `val` as a string.
412412pub fn string(self: *Serializer, val: []const u8, options: StringOptions) Writer.Error!void {
413 if (!options.escape_non_ascii) {
414 return try self.writer.print("{q}", .{val});
415 }
416
413417 try self.writer.writeByte('"');
414 // Batch write sequences of "raw" bytes (printable ASCII or non-escaped non-ASCII) for performance.
415 // `val[start..i]` contains pending raw bytes to write.
416 var start: usize = 0;
417418 var i: usize = 0;
418419 while (i < val.len) {
419420 const byte = val[i];
420 // Check if this byte can be written as-is
421 const is_raw = switch (byte) {
422 ' ', '!', '#'...'[', ']'...'~' => true,
423 0x80...0xFF => !options.escape_non_ascii,
424 else => false,
425 };
426 if (is_raw) {
427 i += 1;
428 continue;
429 }
430 // Flush pending raw bytes
431 try self.writer.writeAll(val[start..i]);
432 // Handle the special character
421
433422 if (byte >= 0x80) {
434 // Decode UTF-8 sequence and write the codepoint
435 const ulen = std.unicode.utf8ByteSequenceLength(byte) catch unreachable;
436 const codepoint = std.unicode.utf8Decode(val[i..][0..ulen]) catch unreachable;
437 // InvalidCodepoint cannot occur from valid UTF-8
438 self.writeCodepoint(codepoint, .{
439 .escape_non_ascii = options.escape_non_ascii,
440 .quote_style = .double,
441 }) catch unreachable;
442 i += ulen;
443 } else {
444 // ASCII character that needs escaping
445 self.writeCodepoint(byte, .{
446 .escape_non_ascii = options.escape_non_ascii,
447 .quote_style = .double,
448 }) catch unreachable; // InvalidCodepoint cannot occur for valid ASCII values
449 i += 1;
423 if (std.unicode.utf8ByteSequenceLength(byte)) |ulen| utf8: {
424 if (val[i..].len < ulen) {
425 // Truncated UTF-8 sequence
426 break :utf8;
427 }
428 const codepoint = std.unicode.utf8Decode(val[i..][0..ulen]) catch break :utf8;
429 self.writeCodepoint(codepoint, .{
430 .escape_non_ascii = true,
431 .quote_style = .double,
432 }) catch unreachable;
433 i += ulen;
434 continue;
435 } else |err| switch (err) {
436 error.Utf8InvalidStartByte => {},
437 }
450438 }
451 start = i;
439
440 try std.zig.stringEscape(&.{byte}, self.writer);
441 i += 1;
452442 }
453443
454 try self.writer.writeAll(val[start..]);
455444 try self.writer.writeByte('"');
456445}
457446
447test string {
448 try testString("\"foobar\"", "foobar", .{});
449 try testString("\"€\"", "€", .{});
450 try testString("\"\\u{20ac}\"", "€", .{ .escape_non_ascii = true });
451 try testString("\"ÿ\"", "ÿ", .{});
452 try testString("\"\\u{ff}\"", "ÿ", .{ .escape_non_ascii = true });
453 try testString("\"\\xff\"", &.{0xff}, .{});
454 try testString("\"\\xff\"", &.{0xff}, .{ .escape_non_ascii = true });
455
456 // Truncated UTF-8 sequence (0xe2 starts a 3-byte sequence, 0x80 is a valid continuation byte)
457 try testString("\"\\xe2\\x80\"", &.{ 0xe2, 0x80 }, .{});
458 try testString("\"\\xe2\\x80\"", &.{ 0xe2, 0x80 }, .{ .escape_non_ascii = true });
459}
460
461fn testString(expected: []const u8, input: []const u8, options: StringOptions) !void {
462 var allocating: Writer.Allocating = .init(std.testing.allocator);
463 defer allocating.deinit();
464 var s: Serializer = .{ .writer = &allocating.writer };
465
466 try s.string(input, options);
467 try std.testing.expectEqualSlices(u8, expected, allocating.written());
468}
469
470test "fuzz string non-escaping" {
471 try std.testing.fuzz(StringOptions{ .escape_non_ascii = false }, fuzzString, .{});
472}
473
474test "fuzz string escaping" {
475 try std.testing.fuzz(StringOptions{ .escape_non_ascii = true }, fuzzString, .{});
476}
477
478fn fuzzString(options: StringOptions, smith: *std.testing.Smith) !void {
479 var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
480 defer arena.deinit();
481 const allocator = arena.allocator();
482
483 var allocating: Writer.Allocating = .init(allocator);
484 defer allocating.deinit();
485 var s: Serializer = .{ .writer = &allocating.writer };
486
487 var buf: [0x100]u8 = undefined;
488 const input_len = smith.slice(&buf);
489 const input = buf[0..input_len];
490 try s.string(input, options);
491
492 const encoded = try allocator.dupeSentinel(u8, allocating.written(), 0);
493 defer allocator.free(encoded);
494 const actual = try std.zon.parse.fromSliceAlloc([]const u8, allocator, encoded, null, .{});
495 defer std.zon.parse.free(allocator, actual);
496 try std.testing.expectEqualSlices(u8, input, actual);
497}
498
458499/// Options for formatting multiline strings.
459500pub const MultilineStringOptions = struct {
460501 /// If top level is true, whitespace before and after the multiline string is elided.