authorgravatar for ben@benburkert.comBen Burkert <ben@benburkert.com> 2026-09-30 04:22:13+02:00
committergravatar for squeek502@hotmail.comRyan Liptak <squeek502@hotmail.com> 2026-09-30 04:22:13+02:00
log087ee6f4efce2bb635c11282a43b0014e56f366f
tree603fba260af531be36e9a38e6a9cdc67b2609154
parentbbb48690de0d320e3a3d6436d28acfb8029f80c5

zon: fix serialization of invalid UTF-8 (#36754)

The `std.zon.Serializer.string` method would previously panic with `attempt to unwrap error: Utf8InvalidStartByte` when invoked with a string parameter containing invalid UTF-8 while the `escape_non_ascii` option was enabled, and also allow invalid UTF-8 to pass through unescaped when `escape_non_ascii` was disabled. This commit makes invalid UTF-8 get escaped into hex sequences (e.g. "\xff") regardless of the `escape_non_ascii` setting. Related: #36923 Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36754 Reviewed-by: Ryan Liptak <squeek502@noreply.codeberg.org>

1 files changed, 75 insertions(+), 34 deletions(-)

lib/std/zon/Serializer.zig+75-34
...@@ -410,51 +410,92 @@ pub const StringOptions = struct {...@@ -410,51 +410,92 @@ pub const StringOptions = struct {
410410
411/// Like `value`, but always serializes `val` as a string.411/// Like `value`, but always serializes `val` as a string.
412pub fn string(self: *Serializer, val: []const u8, options: StringOptions) Writer.Error!void {412pub fn string(self: *Serializer, val: []const u8, options: StringOptions) Writer.Error!void {
413 if (!options.escape_non_ascii) {
414 return try self.writer.print("{q}", .{val});
415 }
416
413 try self.writer.writeByte('"');417 try self.writer.writeByte('"');
414 // Batch write sequences of "raw" bytes (printable ASCII or non-escaped non-ASCII) for performance.
415 // `val[start..i]` contains pending raw bytes to write.
416 var start: usize = 0;
417 var i: usize = 0;418 var i: usize = 0;
418 while (i < val.len) {419 while (i < val.len) {
419 const byte = val[i];420 const byte = val[i];
420 // Check if this byte can be written as-is421
421 const is_raw = switch (byte) {
422 ' ', '!', '#'...'[', ']'...'~' => true,
423 0x80...0xFF => !options.escape_non_ascii,
424 else => false,
425 };
426 if (is_raw) {
427 i += 1;
428 continue;
429 }
430 // Flush pending raw bytes
431 try self.writer.writeAll(val[start..i]);
432 // Handle the special character
433 if (byte >= 0x80) {422 if (byte >= 0x80) {
434 // Decode UTF-8 sequence and write the codepoint423 if (std.unicode.utf8ByteSequenceLength(byte)) |ulen| utf8: {
435 const ulen = std.unicode.utf8ByteSequenceLength(byte) catch unreachable;424 if (val[i..].len < ulen) {
436 const codepoint = std.unicode.utf8Decode(val[i..][0..ulen]) catch unreachable;425 // Truncated UTF-8 sequence
437 // InvalidCodepoint cannot occur from valid UTF-8426 break :utf8;
438 self.writeCodepoint(codepoint, .{427 }
439 .escape_non_ascii = options.escape_non_ascii,428 const codepoint = std.unicode.utf8Decode(val[i..][0..ulen]) catch break :utf8;
440 .quote_style = .double,429 self.writeCodepoint(codepoint, .{
441 }) catch unreachable;430 .escape_non_ascii = true,
442 i += ulen;431 .quote_style = .double,
443 } else {432 }) catch unreachable;
444 // ASCII character that needs escaping433 i += ulen;
445 self.writeCodepoint(byte, .{434 continue;
446 .escape_non_ascii = options.escape_non_ascii,435 } else |err| switch (err) {
447 .quote_style = .double,436 error.Utf8InvalidStartByte => {},
448 }) catch unreachable; // InvalidCodepoint cannot occur for valid ASCII values437 }
449 i += 1;
450 }438 }
451 start = i;439
440 try std.zig.stringEscape(&.{byte}, self.writer);
441 i += 1;
452 }442 }
453443
454 try self.writer.writeAll(val[start..]);
455 try self.writer.writeByte('"');444 try self.writer.writeByte('"');
456}445}
457446
447test string {
448 try testString("\"foobar\"", "foobar", .{});
449 try testString("\"€\"", "€", .{});
450 try testString("\"\\u{20ac}\"", "€", .{ .escape_non_ascii = true });
451 try testString("\"ÿ\"", "ÿ", .{});
452 try testString("\"\\u{ff}\"", "ÿ", .{ .escape_non_ascii = true });
453 try testString("\"\\xff\"", &.{0xff}, .{});
454 try testString("\"\\xff\"", &.{0xff}, .{ .escape_non_ascii = true });
455
456 // Truncated UTF-8 sequence (0xe2 starts a 3-byte sequence, 0x80 is a valid continuation byte)
457 try testString("\"\\xe2\\x80\"", &.{ 0xe2, 0x80 }, .{});
458 try testString("\"\\xe2\\x80\"", &.{ 0xe2, 0x80 }, .{ .escape_non_ascii = true });
459}
460
461fn testString(expected: []const u8, input: []const u8, options: StringOptions) !void {
462 var allocating: Writer.Allocating = .init(std.testing.allocator);
463 defer allocating.deinit();
464 var s: Serializer = .{ .writer = &allocating.writer };
465
466 try s.string(input, options);
467 try std.testing.expectEqualSlices(u8, expected, allocating.written());
468}
469
470test "fuzz string non-escaping" {
471 try std.testing.fuzz(StringOptions{ .escape_non_ascii = false }, fuzzString, .{});
472}
473
474test "fuzz string escaping" {
475 try std.testing.fuzz(StringOptions{ .escape_non_ascii = true }, fuzzString, .{});
476}
477
478fn fuzzString(options: StringOptions, smith: *std.testing.Smith) !void {
479 var arena: std.heap.ArenaAllocator = .init(std.testing.allocator);
480 defer arena.deinit();
481 const allocator = arena.allocator();
482
483 var allocating: Writer.Allocating = .init(allocator);
484 defer allocating.deinit();
485 var s: Serializer = .{ .writer = &allocating.writer };
486
487 var buf: [0x100]u8 = undefined;
488 const input_len = smith.slice(&buf);
489 const input = buf[0..input_len];
490 try s.string(input, options);
491
492 const encoded = try allocator.dupeSentinel(u8, allocating.written(), 0);
493 defer allocator.free(encoded);
494 const actual = try std.zon.parse.fromSliceAlloc([]const u8, allocator, encoded, null, .{});
495 defer std.zon.parse.free(allocator, actual);
496 try std.testing.expectEqualSlices(u8, input, actual);
497}
498
458/// Options for formatting multiline strings.499/// Options for formatting multiline strings.
459pub const MultilineStringOptions = struct {500pub const MultilineStringOptions = struct {
460 /// If top level is true, whitespace before and after the multiline string is elided.501 /// If top level is true, whitespace before and after the multiline string is elided.