From 67e6df4313a982fa08ab6ec6de1da427aaf83409 Mon Sep 17 00:00:00 2001 From: mlugg Date: Fri, 18 Jul 2025 09:53:57 +0100 Subject: [PATCH 01/26] tests: remove more old async tests The rejection of #6025 indicates that if stackless coroutines return to Zig, they will look quite different; see #23446 for the working draft proposal for their return (though it will definitely be tweaked before being accepted). Some of this test coverage was deleted in 40d11cc, but because stackless coroutines will take on a new form if re-introduced, I anticipate that essentially *none* of this coverage will be relevant. Of course, if it for some reason is, we can always grab it from the Git history. --- test/behavior/type_info.zig | 14 ----------- .../async/Frame_of_generic_function.zig | 14 ----------- .../async/bad_alignment_in_asynccall.zig | 13 ---------- .../async/exported_async_function.zig | 7 ------ ..._called_outside_of_function_definition.zig | 11 --------- .../frame_causes_function_to_be_async.zig | 13 ---------- ...eventually_is_inferred_to_become_async.zig | 15 ------------ ...c_function_pointer_passed_to_asyncCall.zig | 13 ---------- ...bad_implicit_casting_of_anyframe_types.zig | 24 ------------------- ...g_type_for_argument_tuple_to_asyncCall.zig | 14 ----------- ...suspend function call, callee suspends.zig | 20 ---------------- 11 files changed, 158 deletions(-) delete mode 100644 test/cases/compile_errors/async/Frame_of_generic_function.zig delete mode 100644 test/cases/compile_errors/async/bad_alignment_in_asynccall.zig delete mode 100644 test/cases/compile_errors/async/exported_async_function.zig delete mode 100644 test/cases/compile_errors/async/frame_called_outside_of_function_definition.zig delete mode 100644 test/cases/compile_errors/async/frame_causes_function_to_be_async.zig delete mode 100644 test/cases/compile_errors/async/non-async_function_pointer_eventually_is_inferred_to_become_async.zig delete mode 100644 test/cases/compile_errors/async/non_async_function_pointer_passed_to_asyncCall.zig delete mode 100644 test/cases/compile_errors/async/prevent_bad_implicit_casting_of_anyframe_types.zig delete mode 100644 test/cases/compile_errors/async/wrong_type_for_argument_tuple_to_asyncCall.zig delete mode 100644 test/cases/safety/nosuspend function call, callee suspends.zig diff --git a/test/behavior/type_info.zig b/test/behavior/type_info.zig index f19d90696be0cfcaed6c71dad13f5776ab9433c7..aea8cbb6aa21924da9f4395d98abd8babd8f0f9b 100644 --- a/test/behavior/type_info.zig +++ b/test/behavior/type_info.zig @@ -539,20 +539,6 @@ fn add(a: i32, b: i32) i32 { return a + b; } -test "type info for async frames" { - if (true) { - // https://github.com/ziglang/zig/issues/6025 - return error.SkipZigTest; - } - - switch (@typeInfo(@Frame(add))) { - .frame => |frame| { - try expect(@as(@TypeOf(add), @ptrCast(frame.function)) == add); - }, - else => unreachable, - } -} - test "Declarations are returned in declaration order" { if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; diff --git a/test/cases/compile_errors/async/Frame_of_generic_function.zig b/test/cases/compile_errors/async/Frame_of_generic_function.zig deleted file mode 100644 index af0fb5c72edeaf840439695f1530312357cf4d55..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/Frame_of_generic_function.zig +++ /dev/null @@ -1,14 +0,0 @@ -export fn entry() void { - var frame: @Frame(func) = undefined; - _ = &frame; -} -fn func(comptime T: type) void { - var x: T = undefined; - _ = &x; -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:2:16: error: @Frame() of generic function diff --git a/test/cases/compile_errors/async/bad_alignment_in_asynccall.zig b/test/cases/compile_errors/async/bad_alignment_in_asynccall.zig deleted file mode 100644 index c30815e7e080c85a48a8b27d250c0bb2ee691515..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/bad_alignment_in_asynccall.zig +++ /dev/null @@ -1,13 +0,0 @@ -export fn entry() void { - var ptr: fn () callconv(.@"async") void = func; - var bytes: [64]u8 = undefined; - _ = @asyncCall(&bytes, {}, ptr, .{}); - _ = &ptr; -} -fn func() callconv(.@"async") void {} - -// error -// backend=stage1 -// target=aarch64-linux-none -// -// tmp.zig:4:21: error: expected type '[]align(8) u8', found '*[64]u8' diff --git a/test/cases/compile_errors/async/exported_async_function.zig b/test/cases/compile_errors/async/exported_async_function.zig deleted file mode 100644 index c3be7d4b8004ec64812f2d5556c0f6c01d541beb..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/exported_async_function.zig +++ /dev/null @@ -1,7 +0,0 @@ -export fn foo() callconv(.@"async") void {} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:1:1: error: exported function cannot be async diff --git a/test/cases/compile_errors/async/frame_called_outside_of_function_definition.zig b/test/cases/compile_errors/async/frame_called_outside_of_function_definition.zig deleted file mode 100644 index d14099815287f35321510c87104e0a5831a970a6..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/frame_called_outside_of_function_definition.zig +++ /dev/null @@ -1,11 +0,0 @@ -var handle_undef: anyframe = undefined; -var handle_dummy: anyframe = @frame(); -export fn entry() bool { - return handle_undef == handle_dummy; -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:2:30: error: @frame() called outside of function definition diff --git a/test/cases/compile_errors/async/frame_causes_function_to_be_async.zig b/test/cases/compile_errors/async/frame_causes_function_to_be_async.zig deleted file mode 100644 index f8493b08b2c9e3374ee9c5c8297601c397964dea..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/frame_causes_function_to_be_async.zig +++ /dev/null @@ -1,13 +0,0 @@ -export fn entry() void { - func(); -} -fn func() void { - _ = @frame(); -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:1:1: error: function with calling convention 'C' cannot be async -// tmp.zig:5:9: note: @frame() causes function to be async diff --git a/test/cases/compile_errors/async/non-async_function_pointer_eventually_is_inferred_to_become_async.zig b/test/cases/compile_errors/async/non-async_function_pointer_eventually_is_inferred_to_become_async.zig deleted file mode 100644 index e18b4200289d80343d38cee2da9101f81916c0f5..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/non-async_function_pointer_eventually_is_inferred_to_become_async.zig +++ /dev/null @@ -1,15 +0,0 @@ -export fn a() void { - var non_async_fn: fn () void = undefined; - non_async_fn = func; -} -fn func() void { - suspend {} -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:5:1: error: 'func' cannot be async -// tmp.zig:3:20: note: required to be non-async here -// tmp.zig:6:5: note: suspends here diff --git a/test/cases/compile_errors/async/non_async_function_pointer_passed_to_asyncCall.zig b/test/cases/compile_errors/async/non_async_function_pointer_passed_to_asyncCall.zig deleted file mode 100644 index b62524f6de11afb5be59ceb2a5d1ba75c6edf277..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/non_async_function_pointer_passed_to_asyncCall.zig +++ /dev/null @@ -1,13 +0,0 @@ -export fn entry() void { - var ptr = afunc; - var bytes: [100]u8 align(16) = undefined; - _ = @asyncCall(&bytes, {}, ptr, .{}); - _ = &ptr; -} -fn afunc() void {} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:4:32: error: expected async function, found 'fn () void' diff --git a/test/cases/compile_errors/async/prevent_bad_implicit_casting_of_anyframe_types.zig b/test/cases/compile_errors/async/prevent_bad_implicit_casting_of_anyframe_types.zig deleted file mode 100644 index 6ab99bf00dc0f52d613a8d7a399cc1c5c38f55de..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/prevent_bad_implicit_casting_of_anyframe_types.zig +++ /dev/null @@ -1,24 +0,0 @@ -export fn a() void { - var x: anyframe = undefined; - var y: anyframe->i32 = x; - _ = .{ &x, &y }; -} -export fn b() void { - var x: i32 = undefined; - var y: anyframe->i32 = x; - _ = .{ &x, &y }; -} -export fn c() void { - var x: @Frame(func) = undefined; - var y: anyframe->i32 = &x; - _ = .{ &x, &y }; -} -fn func() void {} - -// error -// backend=stage1 -// target=native -// -// :3:28: error: expected type 'anyframe->i32', found 'anyframe' -// :8:28: error: expected type 'anyframe->i32', found 'i32' -// tmp.zig:13:29: error: expected type 'anyframe->i32', found '*@Frame(func)' diff --git a/test/cases/compile_errors/async/wrong_type_for_argument_tuple_to_asyncCall.zig b/test/cases/compile_errors/async/wrong_type_for_argument_tuple_to_asyncCall.zig deleted file mode 100644 index 7a9be0a8cc6e9123a0beb597d4be2683aa7546ff..0000000000000000000000000000000000000000 --- a/test/cases/compile_errors/async/wrong_type_for_argument_tuple_to_asyncCall.zig +++ /dev/null @@ -1,14 +0,0 @@ -export fn entry1() void { - var frame: @Frame(foo) = undefined; - @asyncCall(&frame, {}, foo, {}); -} - -fn foo() i32 { - return 0; -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:3:33: error: expected tuple or struct, found 'void' diff --git a/test/cases/safety/nosuspend function call, callee suspends.zig b/test/cases/safety/nosuspend function call, callee suspends.zig deleted file mode 100644 index 50f457f31458024c555d3a4c374ffb6e05cd67ac..0000000000000000000000000000000000000000 --- a/test/cases/safety/nosuspend function call, callee suspends.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); - -pub fn panic(message: []const u8, stack_trace: ?*std.builtin.StackTrace, _: ?usize) noreturn { - _ = message; - _ = stack_trace; - std.process.exit(0); -} -pub fn main() !void { - _ = nosuspend add(101, 100); - return error.TestFailed; -} -fn add(a: i32, b: i32) i32 { - if (a > 100) { - suspend {} - } - return a + b; -} -// run -// backend=stage1 -// target=native -- 2.54.0 From 4e6a04929d22015c85780f08fe46d6378ae82b66 Mon Sep 17 00:00:00 2001 From: Alexandre Date: Fri, 18 Jul 2025 17:45:45 -0400 Subject: [PATCH 02/26] Changed u64 to usize to fix #24208 --- lib/std/hash/RapidHash.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/hash/RapidHash.zig b/lib/std/hash/RapidHash.zig index 030c570df9f4e5714db61484b7f113019ab14a13..bb1b45bcc42dbcae74eeffab515614fb48034ea0 100644 --- a/lib/std/hash/RapidHash.zig +++ b/lib/std/hash/RapidHash.zig @@ -20,7 +20,7 @@ pub fn hash(seed: u64, input: []const u8) u64 { if (len <= 16) { if (len >= 4) { - const d: u64 = ((len & 24) >> @intCast(len >> 3)); + const d: usize = ((len & 24) >> @intCast(len >> 3)); const e = len - 4; a = (r32(k) << 32) | r32(k[e..]); b = ((r32(k[d..]) << 32) | r32(k[(e - d)..])); -- 2.54.0 From a0d16829215f56fe42548f1014cba0f2df5a1609 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 11:49:33 -0700 Subject: [PATCH 03/26] std.hash.RapidHash: remove Its design keeps evolving. See https://github.com/Nicoshev/rapidhash/releases It's great to see the design improving, but over time, this will lead to code rot; versions that aren't widely used but would still have to live in the standard library forever and be maintained. Better to be maintained as an external dependency that applications can opt into. Then, in a few years, if a version proves to be stable and widely adopted, it could be considered for inclusion in the standard library. --- lib/std/hash.zig | 2 - lib/std/hash/RapidHash.zig | 125 ------------------------------------- lib/std/hash/benchmark.zig | 6 -- 3 files changed, 133 deletions(-) delete mode 100644 lib/std/hash/RapidHash.zig diff --git a/lib/std/hash.zig b/lib/std/hash.zig index e0eca9e39173fab1f90188eacd0c123f4898901b..5f1697a236167d13aef5f1c8100516aecbfdbd8e 100644 --- a/lib/std/hash.zig +++ b/lib/std/hash.zig @@ -32,8 +32,6 @@ pub const CityHash64 = cityhash.CityHash64; const wyhash = @import("hash/wyhash.zig"); pub const Wyhash = wyhash.Wyhash; -pub const RapidHash = @import("hash/RapidHash.zig"); - const xxhash = @import("hash/xxhash.zig"); pub const XxHash3 = xxhash.XxHash3; pub const XxHash64 = xxhash.XxHash64; diff --git a/lib/std/hash/RapidHash.zig b/lib/std/hash/RapidHash.zig deleted file mode 100644 index bb1b45bcc42dbcae74eeffab515614fb48034ea0..0000000000000000000000000000000000000000 --- a/lib/std/hash/RapidHash.zig +++ /dev/null @@ -1,125 +0,0 @@ -const std = @import("std"); - -const readInt = std.mem.readInt; -const assert = std.debug.assert; -const expect = std.testing.expect; -const expectEqual = std.testing.expectEqual; - -const RAPID_SEED: u64 = 0xbdd89aa982704029; -const RAPID_SECRET: [3]u64 = .{ 0x2d358dccaa6c78a5, 0x8bb84b93962eacc9, 0x4b33a62ed433d4a3 }; - -pub fn hash(seed: u64, input: []const u8) u64 { - const sc = RAPID_SECRET; - const len = input.len; - var a: u64 = 0; - var b: u64 = 0; - var k = input; - var is: [3]u64 = .{ seed, 0, 0 }; - - is[0] ^= mix(seed ^ sc[0], sc[1]) ^ len; - - if (len <= 16) { - if (len >= 4) { - const d: usize = ((len & 24) >> @intCast(len >> 3)); - const e = len - 4; - a = (r32(k) << 32) | r32(k[e..]); - b = ((r32(k[d..]) << 32) | r32(k[(e - d)..])); - } else if (len > 0) - a = (@as(u64, k[0]) << 56) | (@as(u64, k[len >> 1]) << 32) | @as(u64, k[len - 1]); - } else { - var remain = len; - if (len > 48) { - is[1] = is[0]; - is[2] = is[0]; - while (remain >= 96) { - inline for (0..6) |i| { - const m1 = r64(k[8 * i * 2 ..]); - const m2 = r64(k[8 * (i * 2 + 1) ..]); - is[i % 3] = mix(m1 ^ sc[i % 3], m2 ^ is[i % 3]); - } - k = k[96..]; - remain -= 96; - } - if (remain >= 48) { - inline for (0..3) |i| { - const m1 = r64(k[8 * i * 2 ..]); - const m2 = r64(k[8 * (i * 2 + 1) ..]); - is[i] = mix(m1 ^ sc[i], m2 ^ is[i]); - } - k = k[48..]; - remain -= 48; - } - - is[0] ^= is[1] ^ is[2]; - } - - if (remain > 16) { - is[0] = mix(r64(k) ^ sc[2], r64(k[8..]) ^ is[0] ^ sc[1]); - if (remain > 32) { - is[0] = mix(r64(k[16..]) ^ sc[2], r64(k[24..]) ^ is[0]); - } - } - - a = r64(input[len - 16 ..]); - b = r64(input[len - 8 ..]); - } - - a ^= sc[1]; - b ^= is[0]; - mum(&a, &b); - return mix(a ^ sc[0] ^ len, b ^ sc[1]); -} - -test "RapidHash.hash" { - const bytes: []const u8 = "abcdefgh" ** 128; - - const sizes: [13]u64 = .{ 0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024 }; - - const outcomes: [13]u64 = .{ - 0x5a6ef77074ebc84b, - 0xc11328477bc0f5d1, - 0x5644ac035e40d569, - 0x347080fbf5fcd81, - 0x56b66b8dc802bcc, - 0xb6bf9055973aac7c, - 0xed56d62eead1e402, - 0xc19072d767da8ffb, - 0x89bb40a9928a4f0d, - 0xe0af7c5e7b6e29fd, - 0x9a3ed35fbedfa11a, - 0x4c684b2119ca19fb, - 0x4b575f5bf25600d6, - }; - - var success: bool = true; - for (sizes, outcomes) |s, e| { - const r = hash(RAPID_SEED, bytes[0..s]); - - expectEqual(e, r) catch |err| { - std.debug.print("Failed on {d}: {!}\n", .{ s, err }); - success = false; - }; - } - try expect(success); -} - -inline fn mum(a: *u64, b: *u64) void { - const r = @as(u128, a.*) * b.*; - a.* = @truncate(r); - b.* = @truncate(r >> 64); -} - -inline fn mix(a: u64, b: u64) u64 { - var copy_a = a; - var copy_b = b; - mum(©_a, ©_b); - return copy_a ^ copy_b; -} - -inline fn r64(p: []const u8) u64 { - return readInt(u64, p[0..8], .little); -} - -inline fn r32(p: []const u8) u64 { - return readInt(u32, p[0..4], .little); -} diff --git a/lib/std/hash/benchmark.zig b/lib/std/hash/benchmark.zig index 980b41b8ae29a0be13b54778196fb7094888ccd6..84d14a35db5e811241e42fdd3a63c6f1d48949ba 100644 --- a/lib/std/hash/benchmark.zig +++ b/lib/std/hash/benchmark.zig @@ -59,12 +59,6 @@ const hashes = [_]Hash{ .ty = hash.crc.Crc32, .name = "crc32", }, - Hash{ - .ty = hash.RapidHash, - .name = "rapidhash", - .has_iterative_api = false, - .init_u64 = 0, - }, Hash{ .ty = hash.CityHash32, .name = "cityhash-32", -- 2.54.0 From a288266f3310e9ba98456c5e968f8ce434be6cc7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 16:02:06 -0700 Subject: [PATCH 04/26] std.Io.Reader: remove aggressive assert from `fill` with `.fixed("")` you should still be able to do `fill(1)` and have it return error.EndOfStream. --- lib/std/Io/Reader.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index f2a1ec7287bc0998a631486078eeafe5d79130c7..87e83ffd679a99dba79e6ec3cd9b615b3815c214 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -990,9 +990,9 @@ pub fn discardDelimiterLimit(r: *Reader, delimiter: u8, limit: Limit) DiscardDel /// Returns `error.EndOfStream` if and only if there are fewer than `n` bytes /// remaining. /// -/// Asserts buffer capacity is at least `n`. +/// If the end of stream is not encountered, asserts buffer capacity is at +/// least `n`. pub fn fill(r: *Reader, n: usize) Error!void { - assert(n <= r.buffer.len); if (r.seek + n <= r.end) { @branchHint(.likely); return; -- 2.54.0 From c30df072bde3794ea5f3794d93f69e13640c6b7a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 19:32:42 -0700 Subject: [PATCH 05/26] std.json: update to new I/O API also do a little bit of namespace cleanup --- lib/std/json.zig | 103 +- lib/std/json/Scanner.zig | 1767 ++++++++++++++++++++++++++++++ lib/std/json/Stringify.zig | 999 +++++++++++++++++ lib/std/json/dynamic.zig | 18 +- lib/std/json/dynamic_test.zig | 40 +- lib/std/json/fmt.zig | 40 - lib/std/json/hashmap_test.zig | 18 +- lib/std/json/scanner.zig | 1776 ------------------------------- lib/std/json/scanner_test.zig | 78 +- lib/std/json/static.zig | 10 +- lib/std/json/static_test.zig | 30 +- lib/std/json/stringify.zig | 772 -------------- lib/std/json/stringify_test.zig | 504 --------- lib/std/json/test.zig | 11 +- 14 files changed, 2921 insertions(+), 3245 deletions(-) create mode 100644 lib/std/json/Scanner.zig create mode 100644 lib/std/json/Stringify.zig delete mode 100644 lib/std/json/fmt.zig delete mode 100644 lib/std/json/scanner.zig delete mode 100644 lib/std/json/stringify.zig delete mode 100644 lib/std/json/stringify_test.zig diff --git a/lib/std/json.zig b/lib/std/json.zig index c0fb064c6a1b64ea971f0aac9f46b70b0b829094..f81ac1cd654385dce6b48c98b31d9162ab061bd7 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -10,8 +10,8 @@ //! The high-level `stringify` serializes a Zig or `Value` type into JSON. const builtin = @import("builtin"); -const testing = @import("std").testing; -const ArrayList = @import("std").ArrayList; +const std = @import("std"); +const testing = std.testing; test Scanner { var scanner = Scanner.initCompleteInput(testing.allocator, "{\"foo\": 123}\n"); @@ -41,11 +41,13 @@ test Value { try testing.expectEqualSlices(u8, "goes", parsed.value.object.get("anything").?.string); } -test writeStream { - var out = ArrayList(u8).init(testing.allocator); +test Stringify { + var out: std.io.Writer.Allocating = .init(testing.allocator); + var write_stream: Stringify = .{ + .writer = &out.writer, + .options = .{ .whitespace = .indent_2 }, + }; defer out.deinit(); - var write_stream = writeStream(out.writer(), .{ .whitespace = .indent_2 }); - defer write_stream.deinit(); try write_stream.beginObject(); try write_stream.objectField("foo"); try write_stream.write(123); @@ -55,16 +57,7 @@ test writeStream { \\ "foo": 123 \\} ; - try testing.expectEqualSlices(u8, expected, out.items); -} - -test stringify { - var out = ArrayList(u8).init(testing.allocator); - defer out.deinit(); - - const T = struct { a: i32, b: []const u8 }; - try stringify(T{ .a = 123, .b = "xy" }, .{}, out.writer()); - try testing.expectEqualSlices(u8, "{\"a\":123,\"b\":\"xy\"}", out.items); + try testing.expectEqualSlices(u8, expected, out.getWritten()); } pub const ObjectMap = @import("json/dynamic.zig").ObjectMap; @@ -73,18 +66,18 @@ pub const Value = @import("json/dynamic.zig").Value; pub const ArrayHashMap = @import("json/hashmap.zig").ArrayHashMap; -pub const validate = @import("json/scanner.zig").validate; -pub const Error = @import("json/scanner.zig").Error; -pub const reader = @import("json/scanner.zig").reader; -pub const default_buffer_size = @import("json/scanner.zig").default_buffer_size; -pub const Token = @import("json/scanner.zig").Token; -pub const TokenType = @import("json/scanner.zig").TokenType; -pub const Diagnostics = @import("json/scanner.zig").Diagnostics; -pub const AllocWhen = @import("json/scanner.zig").AllocWhen; -pub const default_max_value_len = @import("json/scanner.zig").default_max_value_len; -pub const Reader = @import("json/scanner.zig").Reader; -pub const Scanner = @import("json/scanner.zig").Scanner; -pub const isNumberFormattedLikeAnInteger = @import("json/scanner.zig").isNumberFormattedLikeAnInteger; +pub const Scanner = @import("json/Scanner.zig"); +pub const validate = Scanner.validate; +pub const Error = Scanner.Error; +pub const reader = Scanner.reader; +pub const default_buffer_size = Scanner.default_buffer_size; +pub const Token = Scanner.Token; +pub const TokenType = Scanner.TokenType; +pub const Diagnostics = Scanner.Diagnostics; +pub const AllocWhen = Scanner.AllocWhen; +pub const default_max_value_len = Scanner.default_max_value_len; +pub const Reader = Scanner.Reader; +pub const isNumberFormattedLikeAnInteger = Scanner.isNumberFormattedLikeAnInteger; pub const ParseOptions = @import("json/static.zig").ParseOptions; pub const Parsed = @import("json/static.zig").Parsed; @@ -99,27 +92,49 @@ pub const innerParseFromValue = @import("json/static.zig").innerParseFromValue; pub const ParseError = @import("json/static.zig").ParseError; pub const ParseFromValueError = @import("json/static.zig").ParseFromValueError; -pub const StringifyOptions = @import("json/stringify.zig").StringifyOptions; -pub const stringify = @import("json/stringify.zig").stringify; -pub const stringifyMaxDepth = @import("json/stringify.zig").stringifyMaxDepth; -pub const stringifyArbitraryDepth = @import("json/stringify.zig").stringifyArbitraryDepth; -pub const stringifyAlloc = @import("json/stringify.zig").stringifyAlloc; -pub const writeStream = @import("json/stringify.zig").writeStream; -pub const writeStreamMaxDepth = @import("json/stringify.zig").writeStreamMaxDepth; -pub const writeStreamArbitraryDepth = @import("json/stringify.zig").writeStreamArbitraryDepth; -pub const WriteStream = @import("json/stringify.zig").WriteStream; -pub const encodeJsonString = @import("json/stringify.zig").encodeJsonString; -pub const encodeJsonStringChars = @import("json/stringify.zig").encodeJsonStringChars; - -pub const Formatter = @import("json/fmt.zig").Formatter; -pub const fmt = @import("json/fmt.zig").fmt; +pub const Stringify = @import("json/Stringify.zig"); + +/// Returns a formatter that formats the given value using stringify. +pub fn fmt(value: anytype, options: Stringify.Options) Formatter(@TypeOf(value)) { + return Formatter(@TypeOf(value)){ .value = value, .options = options }; +} + +test fmt { + const expectFmt = std.testing.expectFmt; + try expectFmt("123", "{f}", .{fmt(@as(u32, 123), .{})}); + try expectFmt( + \\{"num":927,"msg":"hello","sub":{"mybool":true}} + , "{f}", .{fmt(struct { + num: u32, + msg: []const u8, + sub: struct { + mybool: bool, + }, + }{ + .num = 927, + .msg = "hello", + .sub = .{ .mybool = true }, + }, .{})}); +} + +/// Formats the given value using stringify. +pub fn Formatter(comptime T: type) type { + return struct { + value: T, + options: Stringify.Options, + + pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { + try Stringify.value(self.value, self.options, writer); + } + }; +} test { _ = @import("json/test.zig"); - _ = @import("json/scanner.zig"); + _ = Scanner; _ = @import("json/dynamic.zig"); _ = @import("json/hashmap.zig"); _ = @import("json/static.zig"); - _ = @import("json/stringify.zig"); + _ = Stringify; _ = @import("json/JSONTestSuite_test.zig"); } diff --git a/lib/std/json/Scanner.zig b/lib/std/json/Scanner.zig new file mode 100644 index 0000000000000000000000000000000000000000..b9c3c506a5254f4815a938387f515fe6200b864c --- /dev/null +++ b/lib/std/json/Scanner.zig @@ -0,0 +1,1767 @@ +//! The lowest level parsing API in this package; +//! supports streaming input with a low memory footprint. +//! The memory requirement is `O(d)` where d is the nesting depth of `[]` or `{}` containers in the input. +//! Specifically `d/8` bytes are required for this purpose, +//! with some extra buffer according to the implementation of `std.ArrayList`. +//! +//! This scanner can emit partial tokens; see `std.json.Token`. +//! The input to this class is a sequence of input buffers that you must supply one at a time. +//! Call `feedInput()` with the first buffer, then call `next()` repeatedly until `error.BufferUnderrun` is returned. +//! Then call `feedInput()` again and so forth. +//! Call `endInput()` when the last input buffer has been given to `feedInput()`, either immediately after calling `feedInput()`, +//! or when `error.BufferUnderrun` requests more data and there is no more. +//! Be sure to call `next()` after calling `endInput()` until `Token.end_of_document` has been returned. +//! +//! Notes on standards compliance: https://datatracker.ietf.org/doc/html/rfc8259 +//! * RFC 8259 requires JSON documents be valid UTF-8, +//! but makes an allowance for systems that are "part of a closed ecosystem". +//! I have no idea what that's supposed to mean in the context of a standard specification. +//! This implementation requires inputs to be valid UTF-8. +//! * RFC 8259 contradicts itself regarding whether lowercase is allowed in \u hex digits, +//! but this is probably a bug in the spec, and it's clear that lowercase is meant to be allowed. +//! (RFC 5234 defines HEXDIG to only allow uppercase.) +//! * When RFC 8259 refers to a "character", I assume they really mean a "Unicode scalar value". +//! See http://www.unicode.org/glossary/#unicode_scalar_value . +//! * RFC 8259 doesn't explicitly disallow unpaired surrogate halves in \u escape sequences, +//! but vaguely implies that \u escapes are for encoding Unicode "characters" (i.e. Unicode scalar values?), +//! which would mean that unpaired surrogate halves are forbidden. +//! By contrast ECMA-404 (a competing(/compatible?) JSON standard, which JavaScript's JSON.parse() conforms to) +//! explicitly allows unpaired surrogate halves. +//! This implementation forbids unpaired surrogate halves in \u sequences. +//! If a high surrogate half appears in a \u sequence, +//! then a low surrogate half must immediately follow in \u notation. +//! * RFC 8259 allows implementations to "accept non-JSON forms or extensions". +//! This implementation does not accept any of that. +//! * RFC 8259 allows implementations to put limits on "the size of texts", +//! "the maximum depth of nesting", "the range and precision of numbers", +//! and "the length and character contents of strings". +//! This low-level implementation does not limit these, +//! except where noted above, and except that nesting depth requires memory allocation. +//! Note that this low-level API does not interpret numbers numerically, +//! but simply emits their source form for some higher level code to make sense of. +//! * This low-level implementation allows duplicate object keys, +//! and key/value pairs are emitted in the order they appear in the input. + +const Scanner = @This(); +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArrayList = std.ArrayList; +const assert = std.debug.assert; +const BitStack = std.BitStack; + +state: State = .value, +string_is_object_key: bool = false, +stack: BitStack, +value_start: usize = undefined, +utf16_code_units: [2]u16 = undefined, + +input: []const u8 = "", +cursor: usize = 0, +is_end_of_input: bool = false, +diagnostics: ?*Diagnostics = null, + +/// The allocator is only used to track `[]` and `{}` nesting levels. +pub fn initStreaming(allocator: Allocator) @This() { + return .{ + .stack = BitStack.init(allocator), + }; +} +/// Use this if your input is a single slice. +/// This is effectively equivalent to: +/// ``` +/// initStreaming(allocator); +/// feedInput(complete_input); +/// endInput(); +/// ``` +pub fn initCompleteInput(allocator: Allocator, complete_input: []const u8) @This() { + return .{ + .stack = BitStack.init(allocator), + .input = complete_input, + .is_end_of_input = true, + }; +} +pub fn deinit(self: *@This()) void { + self.stack.deinit(); + self.* = undefined; +} + +pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void { + diagnostics.cursor_pointer = &self.cursor; + self.diagnostics = diagnostics; +} + +/// Call this whenever you get `error.BufferUnderrun` from `next()`. +/// When there is no more input to provide, call `endInput()`. +pub fn feedInput(self: *@This(), input: []const u8) void { + assert(self.cursor == self.input.len); // Not done with the last input slice. + if (self.diagnostics) |diag| { + diag.total_bytes_before_current_input += self.input.len; + // This usually goes "negative" to measure how far before the beginning + // of the new buffer the current line started. + diag.line_start_cursor -%= self.cursor; + } + self.input = input; + self.cursor = 0; + self.value_start = 0; +} +/// Call this when you will no longer call `feedInput()` anymore. +/// This can be called either immediately after the last `feedInput()`, +/// or at any time afterward, such as when getting `error.BufferUnderrun` from `next()`. +/// Don't forget to call `next*()` after `endInput()` until you get `.end_of_document`. +pub fn endInput(self: *@This()) void { + self.is_end_of_input = true; +} + +pub const NextError = Error || Allocator.Error || error{BufferUnderrun}; +pub const AllocError = Error || Allocator.Error || error{ValueTooLong}; +pub const PeekError = Error || error{BufferUnderrun}; +pub const SkipError = Error || Allocator.Error; +pub const AllocIntoArrayListError = AllocError || error{BufferUnderrun}; + +/// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);` +/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. +/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. +pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token { + return self.nextAllocMax(allocator, when, default_max_value_len); +} + +/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. +/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. +pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token { + assert(self.is_end_of_input); // This function is not available in streaming mode. + const token_type = self.peekNextTokenType() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }; + switch (token_type) { + .number, .string => { + var value_list = ArrayList(u8).init(allocator); + errdefer { + value_list.deinit(); + } + if (self.allocNextIntoArrayListMax(&value_list, when, max_value_len) catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }) |slice| { + return if (token_type == .number) + Token{ .number = slice } + else + Token{ .string = slice }; + } else { + return if (token_type == .number) + Token{ .allocated_number = try value_list.toOwnedSlice() } + else + Token{ .allocated_string = try value_list.toOwnedSlice() }; + } + }, + + // Simple tokens never alloc. + .object_begin, + .object_end, + .array_begin, + .array_end, + .true, + .false, + .null, + .end_of_document, + => return self.next() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }, + } +} + +/// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);` +pub fn allocNextIntoArrayList(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen) AllocIntoArrayListError!?[]const u8 { + return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len); +} +/// The next token type must be either `.number` or `.string`. See `peekNextTokenType()`. +/// When allocation is not necessary with `.alloc_if_needed`, +/// this method returns the content slice from the input buffer, and `value_list` is not touched. +/// When allocation is necessary or with `.alloc_always`, this method concatenates partial tokens into the given `value_list`, +/// and returns `null` once the final `.number` or `.string` token has been written into it. +/// In case of an `error.BufferUnderrun`, partial values will be left in the given value_list. +/// The given `value_list` is never reset by this method, so an `error.BufferUnderrun` situation +/// can be resumed by passing the same array list in again. +/// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type; +/// the caller of this method is expected to know which type of token is being processed. +pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 { + while (true) { + const token = try self.next(); + switch (token) { + // Accumulate partial values. + .partial_number, .partial_string => |slice| { + try appendSlice(value_list, slice, max_value_len); + }, + .partial_string_escaped_1 => |buf| { + try appendSlice(value_list, buf[0..], max_value_len); + }, + .partial_string_escaped_2 => |buf| { + try appendSlice(value_list, buf[0..], max_value_len); + }, + .partial_string_escaped_3 => |buf| { + try appendSlice(value_list, buf[0..], max_value_len); + }, + .partial_string_escaped_4 => |buf| { + try appendSlice(value_list, buf[0..], max_value_len); + }, + + // Return complete values. + .number => |slice| { + if (when == .alloc_if_needed and value_list.items.len == 0) { + // No alloc necessary. + return slice; + } + try appendSlice(value_list, slice, max_value_len); + // The token is complete. + return null; + }, + .string => |slice| { + if (when == .alloc_if_needed and value_list.items.len == 0) { + // No alloc necessary. + return slice; + } + try appendSlice(value_list, slice, max_value_len); + // The token is complete. + return null; + }, + + .object_begin, + .object_end, + .array_begin, + .array_end, + .true, + .false, + .null, + .end_of_document, + => unreachable, // Only .number and .string token types are allowed here. Check peekNextTokenType() before calling this. + + .allocated_number, .allocated_string => unreachable, + } + } +} + +/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. +/// If the next token type is `.object_begin` or `.array_begin`, +/// this function calls `next()` repeatedly until the corresponding `.object_end` or `.array_end` is found. +/// If the next token type is `.number` or `.string`, +/// this function calls `next()` repeatedly until the (non `.partial_*`) `.number` or `.string` token is found. +/// If the next token type is `.true`, `.false`, or `.null`, this function calls `next()` once. +/// The next token type must not be `.object_end`, `.array_end`, or `.end_of_document`; +/// see `peekNextTokenType()`. +pub fn skipValue(self: *@This()) SkipError!void { + assert(self.is_end_of_input); // This function is not available in streaming mode. + switch (self.peekNextTokenType() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }) { + .object_begin, .array_begin => { + self.skipUntilStackHeight(self.stackHeight()) catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }; + }, + .number, .string => { + while (true) { + switch (self.next() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }) { + .partial_number, + .partial_string, + .partial_string_escaped_1, + .partial_string_escaped_2, + .partial_string_escaped_3, + .partial_string_escaped_4, + => continue, + + .number, .string => break, + + else => unreachable, + } + } + }, + .true, .false, .null => { + _ = self.next() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }; + }, + + .object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token. + } +} + +/// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height. +/// Unlike `skipValue()`, this function is available in streaming mode. +pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void { + while (true) { + switch (try self.next()) { + .object_end, .array_end => { + if (self.stackHeight() == terminal_stack_height) break; + }, + .end_of_document => unreachable, + else => continue, + } + } +} + +/// The depth of `{}` or `[]` nesting levels at the current position. +pub fn stackHeight(self: *const @This()) usize { + return self.stack.bit_len; +} + +/// Pre allocate memory to hold the given number of nesting levels. +/// `stackHeight()` up to the given number will not cause allocations. +pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void { + try self.stack.ensureTotalCapacity(height); +} + +/// See `std.json.Token` for documentation of this function. +pub fn next(self: *@This()) NextError!Token { + state_loop: while (true) { + switch (self.state) { + .value => { + switch (try self.skipWhitespaceExpectByte()) { + // Object, Array + '{' => { + try self.stack.push(OBJECT_MODE); + self.cursor += 1; + self.state = .object_start; + return .object_begin; + }, + '[' => { + try self.stack.push(ARRAY_MODE); + self.cursor += 1; + self.state = .array_start; + return .array_begin; + }, + + // String + '"' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + continue :state_loop; + }, + + // Number + '1'...'9' => { + self.value_start = self.cursor; + self.cursor += 1; + self.state = .number_int; + continue :state_loop; + }, + '0' => { + self.value_start = self.cursor; + self.cursor += 1; + self.state = .number_leading_zero; + continue :state_loop; + }, + '-' => { + self.value_start = self.cursor; + self.cursor += 1; + self.state = .number_minus; + continue :state_loop; + }, + + // literal values + 't' => { + self.cursor += 1; + self.state = .literal_t; + continue :state_loop; + }, + 'f' => { + self.cursor += 1; + self.state = .literal_f; + continue :state_loop; + }, + 'n' => { + self.cursor += 1; + self.state = .literal_n; + continue :state_loop; + }, + + else => return error.SyntaxError, + } + }, + + .post_value => { + if (try self.skipWhitespaceCheckEnd()) return .end_of_document; + + const c = self.input[self.cursor]; + if (self.string_is_object_key) { + self.string_is_object_key = false; + switch (c) { + ':' => { + self.cursor += 1; + self.state = .value; + continue :state_loop; + }, + else => return error.SyntaxError, + } + } + + switch (c) { + '}' => { + if (self.stack.pop() != OBJECT_MODE) return error.SyntaxError; + self.cursor += 1; + // stay in .post_value state. + return .object_end; + }, + ']' => { + if (self.stack.pop() != ARRAY_MODE) return error.SyntaxError; + self.cursor += 1; + // stay in .post_value state. + return .array_end; + }, + ',' => { + switch (self.stack.peek()) { + OBJECT_MODE => { + self.state = .object_post_comma; + }, + ARRAY_MODE => { + self.state = .value; + }, + } + self.cursor += 1; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + + .object_start => { + switch (try self.skipWhitespaceExpectByte()) { + '"' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + self.string_is_object_key = true; + continue :state_loop; + }, + '}' => { + self.cursor += 1; + _ = self.stack.pop(); + self.state = .post_value; + return .object_end; + }, + else => return error.SyntaxError, + } + }, + .object_post_comma => { + switch (try self.skipWhitespaceExpectByte()) { + '"' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + self.string_is_object_key = true; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + + .array_start => { + switch (try self.skipWhitespaceExpectByte()) { + ']' => { + self.cursor += 1; + _ = self.stack.pop(); + self.state = .post_value; + return .array_end; + }, + else => { + self.state = .value; + continue :state_loop; + }, + } + }, + + .number_minus => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); + switch (self.input[self.cursor]) { + '0' => { + self.cursor += 1; + self.state = .number_leading_zero; + continue :state_loop; + }, + '1'...'9' => { + self.cursor += 1; + self.state = .number_int; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .number_leading_zero => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(true); + switch (self.input[self.cursor]) { + '.' => { + self.cursor += 1; + self.state = .number_post_dot; + continue :state_loop; + }, + 'e', 'E' => { + self.cursor += 1; + self.state = .number_post_e; + continue :state_loop; + }, + else => { + self.state = .post_value; + return Token{ .number = self.takeValueSlice() }; + }, + } + }, + .number_int => { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + '0'...'9' => continue, + '.' => { + self.cursor += 1; + self.state = .number_post_dot; + continue :state_loop; + }, + 'e', 'E' => { + self.cursor += 1; + self.state = .number_post_e; + continue :state_loop; + }, + else => { + self.state = .post_value; + return Token{ .number = self.takeValueSlice() }; + }, + } + } + return self.endOfBufferInNumber(true); + }, + .number_post_dot => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); + switch (self.input[self.cursor]) { + '0'...'9' => { + self.cursor += 1; + self.state = .number_frac; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .number_frac => { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + '0'...'9' => continue, + 'e', 'E' => { + self.cursor += 1; + self.state = .number_post_e; + continue :state_loop; + }, + else => { + self.state = .post_value; + return Token{ .number = self.takeValueSlice() }; + }, + } + } + return self.endOfBufferInNumber(true); + }, + .number_post_e => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); + switch (self.input[self.cursor]) { + '0'...'9' => { + self.cursor += 1; + self.state = .number_exp; + continue :state_loop; + }, + '+', '-' => { + self.cursor += 1; + self.state = .number_post_e_sign; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .number_post_e_sign => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); + switch (self.input[self.cursor]) { + '0'...'9' => { + self.cursor += 1; + self.state = .number_exp; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .number_exp => { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + '0'...'9' => continue, + else => { + self.state = .post_value; + return Token{ .number = self.takeValueSlice() }; + }, + } + } + return self.endOfBufferInNumber(true); + }, + + .string => { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + 0...0x1f => return error.SyntaxError, // Bare ASCII control code in string. + + // ASCII plain text. + 0x20...('"' - 1), ('"' + 1)...('\\' - 1), ('\\' + 1)...0x7F => continue, + + // Special characters. + '"' => { + const result = Token{ .string = self.takeValueSlice() }; + self.cursor += 1; + self.state = .post_value; + return result; + }, + '\\' => { + const slice = self.takeValueSlice(); + self.cursor += 1; + self.state = .string_backslash; + if (slice.len > 0) return Token{ .partial_string = slice }; + continue :state_loop; + }, + + // UTF-8 validation. + // See http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String + 0xC2...0xDF => { + self.cursor += 1; + self.state = .string_utf8_last_byte; + continue :state_loop; + }, + 0xE0 => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte_guard_against_overlong; + continue :state_loop; + }, + 0xE1...0xEC, 0xEE...0xEF => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte; + continue :state_loop; + }, + 0xED => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte_guard_against_surrogate_half; + continue :state_loop; + }, + 0xF0 => { + self.cursor += 1; + self.state = .string_utf8_third_to_last_byte_guard_against_overlong; + continue :state_loop; + }, + 0xF1...0xF3 => { + self.cursor += 1; + self.state = .string_utf8_third_to_last_byte; + continue :state_loop; + }, + 0xF4 => { + self.cursor += 1; + self.state = .string_utf8_third_to_last_byte_guard_against_too_large; + continue :state_loop; + }, + 0x80...0xC1, 0xF5...0xFF => return error.SyntaxError, // Invalid UTF-8. + } + } + if (self.is_end_of_input) return error.UnexpectedEndOfInput; + const slice = self.takeValueSlice(); + if (slice.len > 0) return Token{ .partial_string = slice }; + return error.BufferUnderrun; + }, + .string_backslash => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + '"', '\\', '/' => { + // Since these characters now represent themselves literally, + // we can simply begin the next plaintext slice here. + self.value_start = self.cursor; + self.cursor += 1; + self.state = .string; + continue :state_loop; + }, + 'b' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{0x08} }; + }, + 'f' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{0x0c} }; + }, + 'n' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{'\n'} }; + }, + 'r' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{'\r'} }; + }, + 't' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{'\t'} }; + }, + 'u' => { + self.cursor += 1; + self.state = .string_backslash_u; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .string_backslash_u => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[0] = @as(u16, c - '0') << 12; + }, + 'A'...'F' => { + self.utf16_code_units[0] = @as(u16, c - 'A' + 10) << 12; + }, + 'a'...'f' => { + self.utf16_code_units[0] = @as(u16, c - 'a' + 10) << 12; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + self.state = .string_backslash_u_1; + continue :state_loop; + }, + .string_backslash_u_1 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[0] |= @as(u16, c - '0') << 8; + }, + 'A'...'F' => { + self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 8; + }, + 'a'...'f' => { + self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 8; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + self.state = .string_backslash_u_2; + continue :state_loop; + }, + .string_backslash_u_2 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[0] |= @as(u16, c - '0') << 4; + }, + 'A'...'F' => { + self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 4; + }, + 'a'...'f' => { + self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 4; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + self.state = .string_backslash_u_3; + continue :state_loop; + }, + .string_backslash_u_3 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[0] |= c - '0'; + }, + 'A'...'F' => { + self.utf16_code_units[0] |= c - 'A' + 10; + }, + 'a'...'f' => { + self.utf16_code_units[0] |= c - 'a' + 10; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + if (std.unicode.utf16IsHighSurrogate(self.utf16_code_units[0])) { + self.state = .string_surrogate_half; + continue :state_loop; + } else if (std.unicode.utf16IsLowSurrogate(self.utf16_code_units[0])) { + return error.SyntaxError; // Unexpected low surrogate half. + } else { + self.value_start = self.cursor; + self.state = .string; + return partialStringCodepoint(self.utf16_code_units[0]); + } + }, + .string_surrogate_half => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + '\\' => { + self.cursor += 1; + self.state = .string_surrogate_half_backslash; + continue :state_loop; + }, + else => return error.SyntaxError, // Expected low surrogate half. + } + }, + .string_surrogate_half_backslash => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 'u' => { + self.cursor += 1; + self.state = .string_surrogate_half_backslash_u; + continue :state_loop; + }, + else => return error.SyntaxError, // Expected low surrogate half. + } + }, + .string_surrogate_half_backslash_u => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 'D', 'd' => { + self.cursor += 1; + self.utf16_code_units[1] = 0xD << 12; + self.state = .string_surrogate_half_backslash_u_1; + continue :state_loop; + }, + else => return error.SyntaxError, // Expected low surrogate half. + } + }, + .string_surrogate_half_backslash_u_1 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + 'C'...'F' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 8; + self.state = .string_surrogate_half_backslash_u_2; + continue :state_loop; + }, + 'c'...'f' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 8; + self.state = .string_surrogate_half_backslash_u_2; + continue :state_loop; + }, + else => return error.SyntaxError, // Expected low surrogate half. + } + }, + .string_surrogate_half_backslash_u_2 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - '0') << 4; + self.state = .string_surrogate_half_backslash_u_3; + continue :state_loop; + }, + 'A'...'F' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 4; + self.state = .string_surrogate_half_backslash_u_3; + continue :state_loop; + }, + 'a'...'f' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 4; + self.state = .string_surrogate_half_backslash_u_3; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .string_surrogate_half_backslash_u_3 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[1] |= c - '0'; + }, + 'A'...'F' => { + self.utf16_code_units[1] |= c - 'A' + 10; + }, + 'a'...'f' => { + self.utf16_code_units[1] |= c - 'a' + 10; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + const code_point = std.unicode.utf16DecodeSurrogatePair(&self.utf16_code_units) catch unreachable; + return partialStringCodepoint(code_point); + }, + + .string_utf8_last_byte => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0xBF => { + self.cursor += 1; + self.state = .string; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_second_to_last_byte => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0xBF => { + self.cursor += 1; + self.state = .string_utf8_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_second_to_last_byte_guard_against_overlong => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0xA0...0xBF => { + self.cursor += 1; + self.state = .string_utf8_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_second_to_last_byte_guard_against_surrogate_half => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0x9F => { + self.cursor += 1; + self.state = .string_utf8_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_third_to_last_byte => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0xBF => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_third_to_last_byte_guard_against_overlong => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x90...0xBF => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_third_to_last_byte_guard_against_too_large => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0x8F => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + + .literal_t => { + switch (try self.expectByte()) { + 'r' => { + self.cursor += 1; + self.state = .literal_tr; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_tr => { + switch (try self.expectByte()) { + 'u' => { + self.cursor += 1; + self.state = .literal_tru; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_tru => { + switch (try self.expectByte()) { + 'e' => { + self.cursor += 1; + self.state = .post_value; + return .true; + }, + else => return error.SyntaxError, + } + }, + .literal_f => { + switch (try self.expectByte()) { + 'a' => { + self.cursor += 1; + self.state = .literal_fa; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_fa => { + switch (try self.expectByte()) { + 'l' => { + self.cursor += 1; + self.state = .literal_fal; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_fal => { + switch (try self.expectByte()) { + 's' => { + self.cursor += 1; + self.state = .literal_fals; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_fals => { + switch (try self.expectByte()) { + 'e' => { + self.cursor += 1; + self.state = .post_value; + return .false; + }, + else => return error.SyntaxError, + } + }, + .literal_n => { + switch (try self.expectByte()) { + 'u' => { + self.cursor += 1; + self.state = .literal_nu; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_nu => { + switch (try self.expectByte()) { + 'l' => { + self.cursor += 1; + self.state = .literal_nul; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_nul => { + switch (try self.expectByte()) { + 'l' => { + self.cursor += 1; + self.state = .post_value; + return .null; + }, + else => return error.SyntaxError, + } + }, + } + unreachable; + } +} + +/// Seeks ahead in the input until the first byte of the next token (or the end of the input) +/// determines which type of token will be returned from the next `next*()` call. +/// This function is idempotent, only advancing past commas, colons, and inter-token whitespace. +pub fn peekNextTokenType(self: *@This()) PeekError!TokenType { + state_loop: while (true) { + switch (self.state) { + .value => { + switch (try self.skipWhitespaceExpectByte()) { + '{' => return .object_begin, + '[' => return .array_begin, + '"' => return .string, + '-', '0'...'9' => return .number, + 't' => return .true, + 'f' => return .false, + 'n' => return .null, + else => return error.SyntaxError, + } + }, + + .post_value => { + if (try self.skipWhitespaceCheckEnd()) return .end_of_document; + + const c = self.input[self.cursor]; + if (self.string_is_object_key) { + self.string_is_object_key = false; + switch (c) { + ':' => { + self.cursor += 1; + self.state = .value; + continue :state_loop; + }, + else => return error.SyntaxError, + } + } + + switch (c) { + '}' => return .object_end, + ']' => return .array_end, + ',' => { + switch (self.stack.peek()) { + OBJECT_MODE => { + self.state = .object_post_comma; + }, + ARRAY_MODE => { + self.state = .value; + }, + } + self.cursor += 1; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + + .object_start => { + switch (try self.skipWhitespaceExpectByte()) { + '"' => return .string, + '}' => return .object_end, + else => return error.SyntaxError, + } + }, + .object_post_comma => { + switch (try self.skipWhitespaceExpectByte()) { + '"' => return .string, + else => return error.SyntaxError, + } + }, + + .array_start => { + switch (try self.skipWhitespaceExpectByte()) { + ']' => return .array_end, + else => { + self.state = .value; + continue :state_loop; + }, + } + }, + + .number_minus, + .number_leading_zero, + .number_int, + .number_post_dot, + .number_frac, + .number_post_e, + .number_post_e_sign, + .number_exp, + => return .number, + + .string, + .string_backslash, + .string_backslash_u, + .string_backslash_u_1, + .string_backslash_u_2, + .string_backslash_u_3, + .string_surrogate_half, + .string_surrogate_half_backslash, + .string_surrogate_half_backslash_u, + .string_surrogate_half_backslash_u_1, + .string_surrogate_half_backslash_u_2, + .string_surrogate_half_backslash_u_3, + => return .string, + + .string_utf8_last_byte, + .string_utf8_second_to_last_byte, + .string_utf8_second_to_last_byte_guard_against_overlong, + .string_utf8_second_to_last_byte_guard_against_surrogate_half, + .string_utf8_third_to_last_byte, + .string_utf8_third_to_last_byte_guard_against_overlong, + .string_utf8_third_to_last_byte_guard_against_too_large, + => return .string, + + .literal_t, + .literal_tr, + .literal_tru, + => return .true, + .literal_f, + .literal_fa, + .literal_fal, + .literal_fals, + => return .false, + .literal_n, + .literal_nu, + .literal_nul, + => return .null, + } + unreachable; + } +} + +const State = enum { + value, + post_value, + + object_start, + object_post_comma, + + array_start, + + number_minus, + number_leading_zero, + number_int, + number_post_dot, + number_frac, + number_post_e, + number_post_e_sign, + number_exp, + + string, + string_backslash, + string_backslash_u, + string_backslash_u_1, + string_backslash_u_2, + string_backslash_u_3, + string_surrogate_half, + string_surrogate_half_backslash, + string_surrogate_half_backslash_u, + string_surrogate_half_backslash_u_1, + string_surrogate_half_backslash_u_2, + string_surrogate_half_backslash_u_3, + + // From http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String + string_utf8_last_byte, // State A + string_utf8_second_to_last_byte, // State B + string_utf8_second_to_last_byte_guard_against_overlong, // State C + string_utf8_second_to_last_byte_guard_against_surrogate_half, // State D + string_utf8_third_to_last_byte, // State E + string_utf8_third_to_last_byte_guard_against_overlong, // State F + string_utf8_third_to_last_byte_guard_against_too_large, // State G + + literal_t, + literal_tr, + literal_tru, + literal_f, + literal_fa, + literal_fal, + literal_fals, + literal_n, + literal_nu, + literal_nul, +}; + +fn expectByte(self: *const @This()) !u8 { + if (self.cursor < self.input.len) { + return self.input[self.cursor]; + } + // No byte. + if (self.is_end_of_input) return error.UnexpectedEndOfInput; + return error.BufferUnderrun; +} + +fn skipWhitespace(self: *@This()) void { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + // Whitespace + ' ', '\t', '\r' => continue, + '\n' => { + if (self.diagnostics) |diag| { + diag.line_number += 1; + // This will count the newline itself, + // which means a straight-forward subtraction will give a 1-based column number. + diag.line_start_cursor = self.cursor; + } + continue; + }, + else => return, + } + } +} + +fn skipWhitespaceExpectByte(self: *@This()) !u8 { + self.skipWhitespace(); + return self.expectByte(); +} + +fn skipWhitespaceCheckEnd(self: *@This()) !bool { + self.skipWhitespace(); + if (self.cursor >= self.input.len) { + // End of buffer. + if (self.is_end_of_input) { + // End of everything. + if (self.stackHeight() == 0) { + // We did it! + return true; + } + return error.UnexpectedEndOfInput; + } + return error.BufferUnderrun; + } + if (self.stackHeight() == 0) return error.SyntaxError; + return false; +} + +fn takeValueSlice(self: *@This()) []const u8 { + const slice = self.input[self.value_start..self.cursor]; + self.value_start = self.cursor; + return slice; +} +fn takeValueSliceMinusTrailingOffset(self: *@This(), trailing_negative_offset: usize) []const u8 { + // Check if the escape sequence started before the current input buffer. + // (The algebra here is awkward to avoid unsigned underflow, + // but it's just making sure the slice on the next line isn't UB.) + if (self.cursor <= self.value_start + trailing_negative_offset) return ""; + const slice = self.input[self.value_start .. self.cursor - trailing_negative_offset]; + // When trailing_negative_offset is non-zero, setting self.value_start doesn't matter, + // because we always set it again while emitting the .partial_string_escaped_*. + self.value_start = self.cursor; + return slice; +} + +fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token { + const slice = self.takeValueSlice(); + if (self.is_end_of_input) { + if (!allow_end) return error.UnexpectedEndOfInput; + self.state = .post_value; + return Token{ .number = slice }; + } + if (slice.len == 0) return error.BufferUnderrun; + return Token{ .partial_number = slice }; +} + +fn endOfBufferInString(self: *@This()) !Token { + if (self.is_end_of_input) return error.UnexpectedEndOfInput; + const slice = self.takeValueSliceMinusTrailingOffset(switch (self.state) { + // Don't include the escape sequence in the partial string. + .string_backslash => 1, + .string_backslash_u => 2, + .string_backslash_u_1 => 3, + .string_backslash_u_2 => 4, + .string_backslash_u_3 => 5, + .string_surrogate_half => 6, + .string_surrogate_half_backslash => 7, + .string_surrogate_half_backslash_u => 8, + .string_surrogate_half_backslash_u_1 => 9, + .string_surrogate_half_backslash_u_2 => 10, + .string_surrogate_half_backslash_u_3 => 11, + + // Include everything up to the cursor otherwise. + .string, + .string_utf8_last_byte, + .string_utf8_second_to_last_byte, + .string_utf8_second_to_last_byte_guard_against_overlong, + .string_utf8_second_to_last_byte_guard_against_surrogate_half, + .string_utf8_third_to_last_byte, + .string_utf8_third_to_last_byte_guard_against_overlong, + .string_utf8_third_to_last_byte_guard_against_too_large, + => 0, + + else => unreachable, + }); + if (slice.len == 0) return error.BufferUnderrun; + return Token{ .partial_string = slice }; +} + +fn partialStringCodepoint(code_point: u21) Token { + var buf: [4]u8 = undefined; + switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) { + 1 => return Token{ .partial_string_escaped_1 = buf[0..1].* }, + 2 => return Token{ .partial_string_escaped_2 = buf[0..2].* }, + 3 => return Token{ .partial_string_escaped_3 = buf[0..3].* }, + 4 => return Token{ .partial_string_escaped_4 = buf[0..4].* }, + else => unreachable, + } +} + +/// Scan the input and check for malformed JSON. +/// On `SyntaxError` or `UnexpectedEndOfInput`, returns `false`. +/// Returns any errors from the allocator as-is, which is unlikely, +/// but can be caused by extreme nesting depth in the input. +pub fn validate(allocator: Allocator, s: []const u8) Allocator.Error!bool { + var scanner = Scanner.initCompleteInput(allocator, s); + defer scanner.deinit(); + + while (true) { + const token = scanner.next() catch |err| switch (err) { + error.SyntaxError, error.UnexpectedEndOfInput => return false, + error.OutOfMemory => return error.OutOfMemory, + error.BufferUnderrun => unreachable, + }; + if (token == .end_of_document) break; + } + + return true; +} + +/// The parsing errors are divided into two categories: +/// * `SyntaxError` is for clearly malformed JSON documents, +/// such as giving an input document that isn't JSON at all. +/// * `UnexpectedEndOfInput` is for signaling that everything's been +/// valid so far, but the input appears to be truncated for some reason. +/// Note that a completely empty (or whitespace-only) input will give `UnexpectedEndOfInput`. +pub const Error = error{ SyntaxError, UnexpectedEndOfInput }; + +/// Used by `json.reader`. +pub const default_buffer_size = 0x1000; + +/// The tokens emitted by `std.json.Scanner` and `std.json.Reader` `.next*()` functions follow this grammar: +/// ``` +/// = .end_of_document +/// = +/// | +/// | +/// | +/// | +/// | .true +/// | .false +/// | .null +/// = .object_begin ( )* .object_end +/// = .array_begin ( )* .array_end +/// = +/// = +/// ``` +/// +/// What you get for `` and `` values depends on which `next*()` method you call: +/// +/// ``` +/// next(): +/// = ( .partial_number )* .number +/// = ( )* .string +/// = +/// | .partial_string +/// | .partial_string_escaped_1 +/// | .partial_string_escaped_2 +/// | .partial_string_escaped_3 +/// | .partial_string_escaped_4 +/// +/// nextAlloc*(..., .alloc_always): +/// = .allocated_number +/// = .allocated_string +/// +/// nextAlloc*(..., .alloc_if_needed): +/// = +/// | .number +/// | .allocated_number +/// = +/// | .string +/// | .allocated_string +/// ``` +/// +/// For all tokens with a `[]const u8`, `[]u8`, or `[n]u8` payload, the payload represents the content of the value. +/// For number values, this is the representation of the number exactly as it appears in the input. +/// For strings, this is the content of the string after resolving escape sequences. +/// +/// For `.allocated_number` and `.allocated_string`, the `[]u8` payloads are allocations made with the given allocator. +/// You are responsible for managing that memory. `json.Reader.deinit()` does *not* free those allocations. +/// +/// The `.partial_*` tokens indicate that a value spans multiple input buffers or that a string contains escape sequences. +/// To get a complete value in memory, you need to concatenate the values yourself. +/// Calling `nextAlloc*()` does this for you, and returns an `.allocated_*` token with the result. +/// +/// For tokens with a `[]const u8` payload, the payload is a slice into the current input buffer. +/// The memory may become undefined during the next call to `json.Scanner.feedInput()` +/// or any `json.Reader` method whose return error set includes `json.Error`. +/// To keep the value persistently, it recommended to make a copy or to use `.alloc_always`, +/// which makes a copy for you. +/// +/// Note that `.number` and `.string` tokens that follow `.partial_*` tokens may have `0` length to indicate that +/// the previously partial value is completed with no additional bytes. +/// (This can happen when the break between input buffers happens to land on the exact end of a value. E.g. `"[1234"`, `"]"`.) +/// `.partial_*` tokens never have `0` length. +/// +/// The recommended strategy for using the different `next*()` methods is something like this: +/// +/// When you're expecting an object key, use `.alloc_if_needed`. +/// You often don't need a copy of the key string to persist; you might just check which field it is. +/// In the case that the key happens to require an allocation, free it immediately after checking it. +/// +/// When you're expecting a meaningful string value (such as on the right of a `:`), +/// use `.alloc_always` in order to keep the value valid throughout parsing the rest of the document. +/// +/// When you're expecting a number value, use `.alloc_if_needed`. +/// You're probably going to be parsing the string representation of the number into a numeric representation, +/// so you need the complete string representation only temporarily. +/// +/// When you're skipping an unrecognized value, use `skipValue()`. +pub const Token = union(enum) { + object_begin, + object_end, + array_begin, + array_end, + + true, + false, + null, + + number: []const u8, + partial_number: []const u8, + allocated_number: []u8, + + string: []const u8, + partial_string: []const u8, + partial_string_escaped_1: [1]u8, + partial_string_escaped_2: [2]u8, + partial_string_escaped_3: [3]u8, + partial_string_escaped_4: [4]u8, + allocated_string: []u8, + + end_of_document, +}; + +/// This is only used in `peekNextTokenType()` and gives a categorization based on the first byte of the next token that will be emitted from a `next*()` call. +pub const TokenType = enum { + object_begin, + object_end, + array_begin, + array_end, + true, + false, + null, + number, + string, + end_of_document, +}; + +/// To enable diagnostics, declare `var diagnostics = Diagnostics{};` then call `source.enableDiagnostics(&diagnostics);` +/// where `source` is either a `std.json.Reader` or a `std.json.Scanner` that has just been initialized. +/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()` +/// to get meaningful information from this. +pub const Diagnostics = struct { + line_number: u64 = 1, + line_start_cursor: usize = @as(usize, @bitCast(@as(isize, -1))), // Start just "before" the input buffer to get a 1-based column for line 1. + total_bytes_before_current_input: u64 = 0, + cursor_pointer: *const usize = undefined, + + /// Starts at 1. + pub fn getLine(self: *const @This()) u64 { + return self.line_number; + } + /// Starts at 1. + pub fn getColumn(self: *const @This()) u64 { + return self.cursor_pointer.* -% self.line_start_cursor; + } + /// Starts at 0. Measures the byte offset since the start of the input. + pub fn getByteOffset(self: *const @This()) u64 { + return self.total_bytes_before_current_input + self.cursor_pointer.*; + } +}; + +/// See the documentation for `std.json.Token`. +pub const AllocWhen = enum { alloc_if_needed, alloc_always }; + +/// For security, the maximum size allocated to store a single string or number value is limited to 4MiB by default. +/// This limit can be specified by calling `nextAllocMax()` instead of `nextAlloc()`. +pub const default_max_value_len = 4 * 1024 * 1024; + +/// All `next*()` methods here handle `error.BufferUnderrun` from `std.json.Scanner`, and then read from the reader. +pub const Reader = struct { + scanner: Scanner, + reader: *std.Io.Reader, + + /// The allocator is only used to track `[]` and `{}` nesting levels. + pub fn init(allocator: Allocator, io_reader: *std.Io.Reader) @This() { + return .{ + .scanner = Scanner.initStreaming(allocator), + .reader = io_reader, + }; + } + pub fn deinit(self: *@This()) void { + self.scanner.deinit(); + self.* = undefined; + } + + /// Calls `std.json.Scanner.enableDiagnostics`. + pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void { + self.scanner.enableDiagnostics(diagnostics); + } + + pub const NextError = std.Io.Reader.Error || Error || Allocator.Error; + pub const SkipError = Reader.NextError; + pub const AllocError = Reader.NextError || error{ValueTooLong}; + pub const PeekError = std.Io.Reader.Error || Error; + + /// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);` + /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. + pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) Reader.AllocError!Token { + return self.nextAllocMax(allocator, when, default_max_value_len); + } + /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. + pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) Reader.AllocError!Token { + const token_type = try self.peekNextTokenType(); + switch (token_type) { + .number, .string => { + var value_list = ArrayList(u8).init(allocator); + errdefer { + value_list.deinit(); + } + if (try self.allocNextIntoArrayListMax(&value_list, when, max_value_len)) |slice| { + return if (token_type == .number) + Token{ .number = slice } + else + Token{ .string = slice }; + } else { + return if (token_type == .number) + Token{ .allocated_number = try value_list.toOwnedSlice() } + else + Token{ .allocated_string = try value_list.toOwnedSlice() }; + } + }, + + // Simple tokens never alloc. + .object_begin, + .object_end, + .array_begin, + .array_end, + .true, + .false, + .null, + .end_of_document, + => return try self.next(), + } + } + + /// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);` + pub fn allocNextIntoArrayList(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen) Reader.AllocError!?[]const u8 { + return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len); + } + /// Calls `std.json.Scanner.allocNextIntoArrayListMax` and handles `error.BufferUnderrun`. + pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) Reader.AllocError!?[]const u8 { + while (true) { + return self.scanner.allocNextIntoArrayListMax(value_list, when, max_value_len) catch |err| switch (err) { + error.BufferUnderrun => { + try self.refillBuffer(); + continue; + }, + else => |other_err| return other_err, + }; + } + } + + /// Like `std.json.Scanner.skipValue`, but handles `error.BufferUnderrun`. + pub fn skipValue(self: *@This()) Reader.SkipError!void { + switch (try self.peekNextTokenType()) { + .object_begin, .array_begin => { + try self.skipUntilStackHeight(self.stackHeight()); + }, + .number, .string => { + while (true) { + switch (try self.next()) { + .partial_number, + .partial_string, + .partial_string_escaped_1, + .partial_string_escaped_2, + .partial_string_escaped_3, + .partial_string_escaped_4, + => continue, + + .number, .string => break, + + else => unreachable, + } + } + }, + .true, .false, .null => { + _ = try self.next(); + }, + + .object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token. + } + } + /// Like `std.json.Scanner.skipUntilStackHeight()` but handles `error.BufferUnderrun`. + pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) Reader.NextError!void { + while (true) { + return self.scanner.skipUntilStackHeight(terminal_stack_height) catch |err| switch (err) { + error.BufferUnderrun => { + try self.refillBuffer(); + continue; + }, + else => |other_err| return other_err, + }; + } + } + + /// Calls `std.json.Scanner.stackHeight`. + pub fn stackHeight(self: *const @This()) usize { + return self.scanner.stackHeight(); + } + /// Calls `std.json.Scanner.ensureTotalStackCapacity`. + pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void { + try self.scanner.ensureTotalStackCapacity(height); + } + + /// See `std.json.Token` for documentation of this function. + pub fn next(self: *@This()) Reader.NextError!Token { + while (true) { + return self.scanner.next() catch |err| switch (err) { + error.BufferUnderrun => { + try self.refillBuffer(); + continue; + }, + else => |other_err| return other_err, + }; + } + } + + /// See `std.json.Scanner.peekNextTokenType()`. + pub fn peekNextTokenType(self: *@This()) Reader.PeekError!TokenType { + while (true) { + return self.scanner.peekNextTokenType() catch |err| switch (err) { + error.BufferUnderrun => { + try self.refillBuffer(); + continue; + }, + else => |other_err| return other_err, + }; + } + } + + fn refillBuffer(self: *@This()) std.Io.Reader.Error!void { + const input = self.reader.peekGreedy(1) catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => return self.scanner.endInput(), + }; + self.reader.toss(input.len); + self.scanner.feedInput(input); + } +}; + +const OBJECT_MODE = 0; +const ARRAY_MODE = 1; + +fn appendSlice(list: *std.ArrayList(u8), buf: []const u8, max_value_len: usize) !void { + const new_len = std.math.add(usize, list.items.len, buf.len) catch return error.ValueTooLong; + if (new_len > max_value_len) return error.ValueTooLong; + try list.appendSlice(buf); +} + +/// For the slice you get from a `Token.number` or `Token.allocated_number`, +/// this function returns true if the number doesn't contain any fraction or exponent components, and is not `-0`. +/// Note, the numeric value encoded by the value may still be an integer, such as `1.0`. +/// This function is meant to give a hint about whether integer parsing or float parsing should be used on the value. +/// This function will not give meaningful results on non-numeric input. +pub fn isNumberFormattedLikeAnInteger(value: []const u8) bool { + if (std.mem.eql(u8, value, "-0")) return false; + return std.mem.indexOfAny(u8, value, ".eE") == null; +} + +test { + _ = @import("./scanner_test.zig"); +} diff --git a/lib/std/json/Stringify.zig b/lib/std/json/Stringify.zig new file mode 100644 index 0000000000000000000000000000000000000000..4d79217a875cfce6a0c4f45cd014e1a4857bcc7c --- /dev/null +++ b/lib/std/json/Stringify.zig @@ -0,0 +1,999 @@ +//! Writes JSON ([RFC8259](https://tools.ietf.org/html/rfc8259)) formatted data +//! to a stream. +//! +//! The sequence of method calls to write JSON content must follow this grammar: +//! ``` +//! = +//! = +//! | +//! | +//! | write +//! | print +//! | +//! = beginObject ( )* endObject +//! = objectField | objectFieldRaw | +//! = beginArray ( )* endArray +//! = beginWriteRaw ( stream.writeAll )* endWriteRaw +//! = beginObjectFieldRaw ( stream.writeAll )* endObjectFieldRaw +//! ``` + +const std = @import("../std.zig"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const ArrayList = std.ArrayList; +const BitStack = std.BitStack; +const Stringify = @This(); +const Writer = std.io.Writer; + +const IndentationMode = enum(u1) { + object = 0, + array = 1, +}; + +writer: *Writer, +options: Options = .{}, +indent_level: usize = 0, +next_punctuation: enum { + the_beginning, + none, + comma, + colon, +} = .the_beginning, + +nesting_stack: switch (safety_checks) { + .checked_to_fixed_depth => |fixed_buffer_size| [(fixed_buffer_size + 7) >> 3]u8, + .assumed_correct => void, +} = switch (safety_checks) { + .checked_to_fixed_depth => @splat(0), + .assumed_correct => {}, +}, + +raw_streaming_mode: if (build_mode_has_safety) + enum { none, value, objectField } +else + void = if (build_mode_has_safety) .none else {}, + +const build_mode_has_safety = switch (@import("builtin").mode) { + .Debug, .ReleaseSafe => true, + .ReleaseFast, .ReleaseSmall => false, +}; + +/// The `safety_checks_hint` parameter determines how much memory is used to enable assertions that the above grammar is being followed, +/// e.g. tripping an assertion rather than allowing `endObject` to emit the final `}` in `[[[]]}`. +/// "Depth" in this context means the depth of nested `[]` or `{}` expressions +/// (or equivalently the amount of recursion on the `` grammar expression above). +/// For example, emitting the JSON `[[[]]]` requires a depth of 3. +/// If `.checked_to_fixed_depth` is used, there is additionally an assertion that the nesting depth never exceeds the given limit. +/// `.checked_to_fixed_depth` embeds the storage required in the `Stringify` struct. +/// `.assumed_correct` requires no space and performs none of these assertions. +/// In `ReleaseFast` and `ReleaseSmall` mode, the given `safety_checks_hint` is ignored and is always treated as `.assumed_correct`. +const safety_checks_hint: union(enum) { + /// Rounded up to the nearest multiple of 8. + checked_to_fixed_depth: usize, + assumed_correct, +} = .{ .checked_to_fixed_depth = 256 }; + +const safety_checks: @TypeOf(safety_checks_hint) = if (build_mode_has_safety) + safety_checks_hint +else + .assumed_correct; + +pub const Error = Writer.Error; + +pub fn beginArray(self: *Stringify) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + try self.valueStart(); + try self.writer.writeByte('['); + try self.pushIndentation(.array); + self.next_punctuation = .none; +} + +pub fn beginObject(self: *Stringify) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + try self.valueStart(); + try self.writer.writeByte('{'); + try self.pushIndentation(.object); + self.next_punctuation = .none; +} + +pub fn endArray(self: *Stringify) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + self.popIndentation(.array); + switch (self.next_punctuation) { + .none => {}, + .comma => { + try self.indent(); + }, + .the_beginning, .colon => unreachable, + } + try self.writer.writeByte(']'); + self.valueDone(); +} + +pub fn endObject(self: *Stringify) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + self.popIndentation(.object); + switch (self.next_punctuation) { + .none => {}, + .comma => { + try self.indent(); + }, + .the_beginning, .colon => unreachable, + } + try self.writer.writeByte('}'); + self.valueDone(); +} + +fn pushIndentation(self: *Stringify, mode: IndentationMode) !void { + switch (safety_checks) { + .checked_to_fixed_depth => { + BitStack.pushWithStateAssumeCapacity(&self.nesting_stack, &self.indent_level, @intFromEnum(mode)); + }, + .assumed_correct => { + self.indent_level += 1; + }, + } +} +fn popIndentation(self: *Stringify, expected_mode: IndentationMode) void { + switch (safety_checks) { + .checked_to_fixed_depth => { + assert(BitStack.popWithState(&self.nesting_stack, &self.indent_level) == @intFromEnum(expected_mode)); + }, + .assumed_correct => { + self.indent_level -= 1; + }, + } +} + +fn indent(self: *Stringify) !void { + var char: u8 = ' '; + const n_chars = switch (self.options.whitespace) { + .minified => return, + .indent_1 => 1 * self.indent_level, + .indent_2 => 2 * self.indent_level, + .indent_3 => 3 * self.indent_level, + .indent_4 => 4 * self.indent_level, + .indent_8 => 8 * self.indent_level, + .indent_tab => blk: { + char = '\t'; + break :blk self.indent_level; + }, + }; + try self.writer.writeByte('\n'); + try self.writer.splatByteAll(char, n_chars); +} + +fn valueStart(self: *Stringify) !void { + if (self.isObjectKeyExpected()) |is_it| assert(!is_it); // Call objectField*(), not write(), for object keys. + return self.valueStartAssumeTypeOk(); +} +fn objectFieldStart(self: *Stringify) !void { + if (self.isObjectKeyExpected()) |is_it| assert(is_it); // Expected write(), not objectField*(). + return self.valueStartAssumeTypeOk(); +} +fn valueStartAssumeTypeOk(self: *Stringify) !void { + assert(!self.isComplete()); // JSON document already complete. + switch (self.next_punctuation) { + .the_beginning => { + // No indentation for the very beginning. + }, + .none => { + // First item in a container. + try self.indent(); + }, + .comma => { + // Subsequent item in a container. + try self.writer.writeByte(','); + try self.indent(); + }, + .colon => { + try self.writer.writeByte(':'); + if (self.options.whitespace != .minified) { + try self.writer.writeByte(' '); + } + }, + } +} +fn valueDone(self: *Stringify) void { + self.next_punctuation = .comma; +} + +// Only when safety is enabled: +fn isObjectKeyExpected(self: *const Stringify) ?bool { + switch (safety_checks) { + .checked_to_fixed_depth => return self.indent_level > 0 and + BitStack.peekWithState(&self.nesting_stack, self.indent_level) == @intFromEnum(IndentationMode.object) and + self.next_punctuation != .colon, + .assumed_correct => return null, + } +} +fn isComplete(self: *const Stringify) bool { + return self.indent_level == 0 and self.next_punctuation == .comma; +} + +/// An alternative to calling `write` that formats a value with `std.fmt`. +/// This function does the usual punctuation and indentation formatting +/// assuming the resulting formatted string represents a single complete value; +/// e.g. `"1"`, `"[]"`, `"[1,2]"`, not `"1,2"`. +/// This function may be useful for doing your own number formatting. +pub fn print(self: *Stringify, comptime fmt: []const u8, args: anytype) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + try self.valueStart(); + try self.writer.print(fmt, args); + self.valueDone(); +} + +test print { + var out_buf: [1024]u8 = undefined; + var out: Writer = .fixed(&out_buf); + + var w: Stringify = .{ .writer = &out, .options = .{ .whitespace = .indent_2 } }; + + try w.beginObject(); + try w.objectField("a"); + try w.print("[ ]", .{}); + try w.objectField("b"); + try w.beginArray(); + try w.print("[{s}] ", .{"[]"}); + try w.print(" {}", .{12345}); + try w.endArray(); + try w.endObject(); + + const expected = + \\{ + \\ "a": [ ], + \\ "b": [ + \\ [[]] , + \\ 12345 + \\ ] + \\} + ; + try std.testing.expectEqualStrings(expected, out.buffered()); +} + +/// An alternative to calling `write` that allows you to write directly to the `.writer` field, e.g. with `.writer.writeAll()`. +/// Call `beginWriteRaw()`, then write a complete value (including any quotes if necessary) directly to the `.writer` field, +/// then call `endWriteRaw()`. +/// This can be useful for streaming very long strings into the output without needing it all buffered in memory. +pub fn beginWriteRaw(self: *Stringify) !void { + if (build_mode_has_safety) { + assert(self.raw_streaming_mode == .none); + self.raw_streaming_mode = .value; + } + try self.valueStart(); +} + +/// See `beginWriteRaw`. +pub fn endWriteRaw(self: *Stringify) void { + if (build_mode_has_safety) { + assert(self.raw_streaming_mode == .value); + self.raw_streaming_mode = .none; + } + self.valueDone(); +} + +/// See `Stringify` for when to call this method. +/// `key` is the string content of the property name. +/// Surrounding quotes will be added and any special characters will be escaped. +/// See also `objectFieldRaw`. +pub fn objectField(self: *Stringify, key: []const u8) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + try self.objectFieldStart(); + try encodeJsonString(key, self.options, self.writer); + self.next_punctuation = .colon; +} +/// See `Stringify` for when to call this method. +/// `quoted_key` is the complete bytes of the key including quotes and any necessary escape sequences. +/// A few assertions are performed on the given value to ensure that the caller of this function understands the API contract. +/// See also `objectField`. +pub fn objectFieldRaw(self: *Stringify, quoted_key: []const u8) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + assert(quoted_key.len >= 2 and quoted_key[0] == '"' and quoted_key[quoted_key.len - 1] == '"'); // quoted_key should be "quoted". + try self.objectFieldStart(); + try self.writer.writeAll(quoted_key); + self.next_punctuation = .colon; +} + +/// In the rare case that you need to write very long object field names, +/// this is an alternative to `objectField` and `objectFieldRaw` that allows you to write directly to the `.writer` field +/// similar to `beginWriteRaw`. +/// Call `endObjectFieldRaw()` when you're done. +pub fn beginObjectFieldRaw(self: *Stringify) !void { + if (build_mode_has_safety) { + assert(self.raw_streaming_mode == .none); + self.raw_streaming_mode = .objectField; + } + try self.objectFieldStart(); +} + +/// See `beginObjectFieldRaw`. +pub fn endObjectFieldRaw(self: *Stringify) void { + if (build_mode_has_safety) { + assert(self.raw_streaming_mode == .objectField); + self.raw_streaming_mode = .none; + } + self.next_punctuation = .colon; +} + +/// Renders the given Zig value as JSON. +/// +/// Supported types: +/// * Zig `bool` -> JSON `true` or `false`. +/// * Zig `?T` -> `null` or the rendering of `T`. +/// * Zig `i32`, `u64`, etc. -> JSON number or string. +/// * When option `emit_nonportable_numbers_as_strings` is true, if the value is outside the range `+-1<<53` (the precise integer range of f64), it is rendered as a JSON string in base 10. Otherwise, it is rendered as JSON number. +/// * Zig floats -> JSON number or string. +/// * If the value cannot be precisely represented by an f64, it is rendered as a JSON string. Otherwise, it is rendered as JSON number. +/// * TODO: Float rendering will likely change in the future, e.g. to remove the unnecessary "e+00". +/// * Zig `[]const u8`, `[]u8`, `*[N]u8`, `@Vector(N, u8)`, and similar -> JSON string. +/// * See `Options.emit_strings_as_arrays`. +/// * If the content is not valid UTF-8, rendered as an array of numbers instead. +/// * Zig `[]T`, `[N]T`, `*[N]T`, `@Vector(N, T)`, and similar -> JSON array of the rendering of each item. +/// * Zig tuple -> JSON array of the rendering of each item. +/// * Zig `struct` -> JSON object with each field in declaration order. +/// * If the struct declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `Stringify`. See `std.json.Value` for an example. +/// * See `Options.emit_null_optional_fields`. +/// * Zig `union(enum)` -> JSON object with one field named for the active tag and a value representing the payload. +/// * If the payload is `void`, then the emitted value is `{}`. +/// * If the union declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `Stringify`. +/// * Zig `enum` -> JSON string naming the active tag. +/// * If the enum declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `Stringify`. +/// * If the enum is non-exhaustive, unnamed values are rendered as integers. +/// * Zig untyped enum literal -> JSON string naming the active tag. +/// * Zig error -> JSON string naming the error. +/// * Zig `*T` -> the rendering of `T`. Note there is no guard against circular-reference infinite recursion. +/// +/// See also alternative functions `print` and `beginWriteRaw`. +/// For writing object field names, use `objectField` instead. +pub fn write(self: *Stringify, v: anytype) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + const T = @TypeOf(v); + switch (@typeInfo(T)) { + .int => { + try self.valueStart(); + if (self.options.emit_nonportable_numbers_as_strings and + (v <= -(1 << 53) or v >= (1 << 53))) + { + try self.writer.print("\"{}\"", .{v}); + } else { + try self.writer.print("{}", .{v}); + } + self.valueDone(); + return; + }, + .comptime_int => { + return self.write(@as(std.math.IntFittingRange(v, v), v)); + }, + .float, .comptime_float => { + if (@as(f64, @floatCast(v)) == v) { + try self.valueStart(); + try self.writer.print("{}", .{@as(f64, @floatCast(v))}); + self.valueDone(); + return; + } + try self.valueStart(); + try self.writer.print("\"{}\"", .{v}); + self.valueDone(); + return; + }, + + .bool => { + try self.valueStart(); + try self.writer.writeAll(if (v) "true" else "false"); + self.valueDone(); + return; + }, + .null => { + try self.valueStart(); + try self.writer.writeAll("null"); + self.valueDone(); + return; + }, + .optional => { + if (v) |payload| { + return try self.write(payload); + } else { + return try self.write(null); + } + }, + .@"enum" => |enum_info| { + if (std.meta.hasFn(T, "jsonStringify")) { + return v.jsonStringify(self); + } + + if (!enum_info.is_exhaustive) { + inline for (enum_info.fields) |field| { + if (v == @field(T, field.name)) { + break; + } + } else { + return self.write(@intFromEnum(v)); + } + } + + return self.stringValue(@tagName(v)); + }, + .enum_literal => { + return self.stringValue(@tagName(v)); + }, + .@"union" => { + if (std.meta.hasFn(T, "jsonStringify")) { + return v.jsonStringify(self); + } + + const info = @typeInfo(T).@"union"; + if (info.tag_type) |UnionTagType| { + try self.beginObject(); + inline for (info.fields) |u_field| { + if (v == @field(UnionTagType, u_field.name)) { + try self.objectField(u_field.name); + if (u_field.type == void) { + // void v is {} + try self.beginObject(); + try self.endObject(); + } else { + try self.write(@field(v, u_field.name)); + } + break; + } + } else { + unreachable; // No active tag? + } + try self.endObject(); + return; + } else { + @compileError("Unable to stringify untagged union '" ++ @typeName(T) ++ "'"); + } + }, + .@"struct" => |S| { + if (std.meta.hasFn(T, "jsonStringify")) { + return v.jsonStringify(self); + } + + if (S.is_tuple) { + try self.beginArray(); + } else { + try self.beginObject(); + } + inline for (S.fields) |Field| { + // don't include void fields + if (Field.type == void) continue; + + var emit_field = true; + + // don't include optional fields that are null when emit_null_optional_fields is set to false + if (@typeInfo(Field.type) == .optional) { + if (self.options.emit_null_optional_fields == false) { + if (@field(v, Field.name) == null) { + emit_field = false; + } + } + } + + if (emit_field) { + if (!S.is_tuple) { + try self.objectField(Field.name); + } + try self.write(@field(v, Field.name)); + } + } + if (S.is_tuple) { + try self.endArray(); + } else { + try self.endObject(); + } + return; + }, + .error_set => return self.stringValue(@errorName(v)), + .pointer => |ptr_info| switch (ptr_info.size) { + .one => switch (@typeInfo(ptr_info.child)) { + .array => { + // Coerce `*[N]T` to `[]const T`. + const Slice = []const std.meta.Elem(ptr_info.child); + return self.write(@as(Slice, v)); + }, + else => { + return self.write(v.*); + }, + }, + .many, .slice => { + if (ptr_info.size == .many and ptr_info.sentinel() == null) + @compileError("unable to stringify type '" ++ @typeName(T) ++ "' without sentinel"); + const slice = if (ptr_info.size == .many) std.mem.span(v) else v; + + if (ptr_info.child == u8) { + // This is a []const u8, or some similar Zig string. + if (!self.options.emit_strings_as_arrays and std.unicode.utf8ValidateSlice(slice)) { + return self.stringValue(slice); + } + } + + try self.beginArray(); + for (slice) |x| { + try self.write(x); + } + try self.endArray(); + return; + }, + else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"), + }, + .array => { + // Coerce `[N]T` to `*const [N]T` (and then to `[]const T`). + return self.write(&v); + }, + .vector => |info| { + const array: [info.len]info.child = v; + return self.write(&array); + }, + else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"), + } + unreachable; +} + +fn stringValue(self: *Stringify, s: []const u8) !void { + try self.valueStart(); + try encodeJsonString(s, self.options, self.writer); + self.valueDone(); +} + +pub const Options = struct { + /// Controls the whitespace emitted. + /// The default `.minified` is a compact encoding with no whitespace between tokens. + /// Any setting other than `.minified` will use newlines, indentation, and a space after each ':'. + /// `.indent_1` means 1 space for each indentation level, `.indent_2` means 2 spaces, etc. + /// `.indent_tab` uses a tab for each indentation level. + whitespace: enum { + minified, + indent_1, + indent_2, + indent_3, + indent_4, + indent_8, + indent_tab, + } = .minified, + + /// Should optional fields with null value be written? + emit_null_optional_fields: bool = true, + + /// Arrays/slices of u8 are typically encoded as JSON strings. + /// This option emits them as arrays of numbers instead. + /// Does not affect calls to `objectField*()`. + emit_strings_as_arrays: bool = false, + + /// Should unicode characters be escaped in strings? + escape_unicode: bool = false, + + /// When true, renders numbers outside the range `+-1<<53` (the precise integer range of f64) as JSON strings in base 10. + emit_nonportable_numbers_as_strings: bool = false, +}; + +/// Writes the given value to the `Writer` writer. +/// See `Stringify` for how the given value is serialized into JSON. +/// The maximum nesting depth of the output JSON document is 256. +pub fn value(v: anytype, options: Options, writer: *Writer) Error!void { + var s: Stringify = .{ .writer = writer, .options = options }; + try s.write(v); +} + +test value { + var out: std.io.Writer.Allocating = .init(std.testing.allocator); + const writer = &out.writer; + defer out.deinit(); + + const T = struct { a: i32, b: []const u8 }; + try value(T{ .a = 123, .b = "xy" }, .{}, writer); + try std.testing.expectEqualSlices(u8, "{\"a\":123,\"b\":\"xy\"}", out.getWritten()); + + try testStringify("9999999999999999", 9999999999999999, .{}); + try testStringify("\"9999999999999999\"", 9999999999999999, .{ .emit_nonportable_numbers_as_strings = true }); + + try testStringify("[1,1]", @as(@Vector(2, u32), @splat(1)), .{}); + try testStringify("\"AA\"", @as(@Vector(2, u8), @splat('A')), .{}); + try testStringify("[65,65]", @as(@Vector(2, u8), @splat('A')), .{ .emit_strings_as_arrays = true }); + + // void field + try testStringify("{\"foo\":42}", struct { + foo: u32, + bar: void = {}, + }{ .foo = 42 }, .{}); + + const Tuple = struct { []const u8, usize }; + try testStringify("[\"foo\",42]", Tuple{ "foo", 42 }, .{}); + + comptime { + testStringify("false", false, .{}) catch unreachable; + const MyStruct = struct { foo: u32 }; + testStringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ + MyStruct{ .foo = 42 }, + MyStruct{ .foo = 100 }, + MyStruct{ .foo = 1000 }, + }, .{}) catch unreachable; + } +} + +/// Calls `value` and stores the result in dynamically allocated memory instead +/// of taking a writer. +/// +/// Caller owns returned memory. +pub fn valueAlloc(gpa: Allocator, v: anytype, options: Options) error{OutOfMemory}![]u8 { + var aw: std.io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + value(v, options, &aw.writer) catch return error.OutOfMemory; + return aw.toOwnedSlice(); +} + +test valueAlloc { + const allocator = std.testing.allocator; + const expected = + \\{"foo":"bar","answer":42,"my_friend":"sammy"} + ; + const actual = try valueAlloc(allocator, .{ .foo = "bar", .answer = 42, .my_friend = "sammy" }, .{}); + defer allocator.free(actual); + + try std.testing.expectEqualStrings(expected, actual); +} + +fn outputUnicodeEscape(codepoint: u21, w: *Writer) Error!void { + if (codepoint <= 0xFFFF) { + // If the character is in the Basic Multilingual Plane (U+0000 through U+FFFF), + // then it may be represented as a six-character sequence: a reverse solidus, followed + // by the lowercase letter u, followed by four hexadecimal digits that encode the character's code point. + try w.writeAll("\\u"); + try w.printInt(codepoint, 16, .lower, .{ .width = 4, .fill = '0' }); + } else { + assert(codepoint <= 0x10FFFF); + // To escape an extended character that is not in the Basic Multilingual Plane, + // the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair. + const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800; + const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00; + try w.writeAll("\\u"); + try w.printInt(high, 16, .lower, .{ .width = 4, .fill = '0' }); + try w.writeAll("\\u"); + try w.printInt(low, 16, .lower, .{ .width = 4, .fill = '0' }); + } +} + +fn outputSpecialEscape(c: u8, writer: *Writer) Error!void { + switch (c) { + '\\' => try writer.writeAll("\\\\"), + '\"' => try writer.writeAll("\\\""), + 0x08 => try writer.writeAll("\\b"), + 0x0C => try writer.writeAll("\\f"), + '\n' => try writer.writeAll("\\n"), + '\r' => try writer.writeAll("\\r"), + '\t' => try writer.writeAll("\\t"), + else => try outputUnicodeEscape(c, writer), + } +} + +/// Write `string` to `writer` as a JSON encoded string. +pub fn encodeJsonString(string: []const u8, options: Options, writer: *Writer) Error!void { + try writer.writeByte('\"'); + try encodeJsonStringChars(string, options, writer); + try writer.writeByte('\"'); +} + +/// Write `chars` to `writer` as JSON encoded string characters. +pub fn encodeJsonStringChars(chars: []const u8, options: Options, writer: *Writer) Error!void { + var write_cursor: usize = 0; + var i: usize = 0; + if (options.escape_unicode) { + while (i < chars.len) : (i += 1) { + switch (chars[i]) { + // normal ascii character + 0x20...0x21, 0x23...0x5B, 0x5D...0x7E => {}, + 0x00...0x1F, '\\', '\"' => { + // Always must escape these. + try writer.writeAll(chars[write_cursor..i]); + try outputSpecialEscape(chars[i], writer); + write_cursor = i + 1; + }, + 0x7F...0xFF => { + try writer.writeAll(chars[write_cursor..i]); + const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable; + const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable; + try outputUnicodeEscape(codepoint, writer); + i += ulen - 1; + write_cursor = i + 1; + }, + } + } + } else { + while (i < chars.len) : (i += 1) { + switch (chars[i]) { + // normal bytes + 0x20...0x21, 0x23...0x5B, 0x5D...0xFF => {}, + 0x00...0x1F, '\\', '\"' => { + // Always must escape these. + try writer.writeAll(chars[write_cursor..i]); + try outputSpecialEscape(chars[i], writer); + write_cursor = i + 1; + }, + } + } + } + try writer.writeAll(chars[write_cursor..chars.len]); +} + +test "json write stream" { + var out_buf: [1024]u8 = undefined; + var out: Writer = .fixed(&out_buf); + var w: Stringify = .{ .writer = &out, .options = .{ .whitespace = .indent_2 } }; + try testBasicWriteStream(&w); +} + +fn testBasicWriteStream(w: *Stringify) !void { + w.writer.end = 0; + + try w.beginObject(); + + try w.objectField("object"); + var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_allocator.deinit(); + try w.write(try getJsonObject(arena_allocator.allocator())); + + try w.objectFieldRaw("\"string\""); + try w.write("This is a string"); + + try w.objectField("array"); + try w.beginArray(); + try w.write("Another string"); + try w.write(@as(i32, 1)); + try w.write(@as(f32, 3.5)); + try w.endArray(); + + try w.objectField("int"); + try w.write(@as(i32, 10)); + + try w.objectField("float"); + try w.write(@as(f32, 3.5)); + + try w.endObject(); + + const expected = + \\{ + \\ "object": { + \\ "one": 1, + \\ "two": 2 + \\ }, + \\ "string": "This is a string", + \\ "array": [ + \\ "Another string", + \\ 1, + \\ 3.5 + \\ ], + \\ "int": 10, + \\ "float": 3.5 + \\} + ; + try std.testing.expectEqualStrings(expected, w.writer.buffered()); +} + +fn getJsonObject(allocator: std.mem.Allocator) !std.json.Value { + var v: std.json.Value = .{ .object = std.json.ObjectMap.init(allocator) }; + try v.object.put("one", std.json.Value{ .integer = @as(i64, @intCast(1)) }); + try v.object.put("two", std.json.Value{ .float = 2.0 }); + return v; +} + +test "stringify null optional fields" { + const MyStruct = struct { + optional: ?[]const u8 = null, + required: []const u8 = "something", + another_optional: ?[]const u8 = null, + another_required: []const u8 = "something else", + }; + try testStringify( + \\{"optional":null,"required":"something","another_optional":null,"another_required":"something else"} + , + MyStruct{}, + .{}, + ); + try testStringify( + \\{"required":"something","another_required":"something else"} + , + MyStruct{}, + .{ .emit_null_optional_fields = false }, + ); +} + +test "stringify basic types" { + try testStringify("false", false, .{}); + try testStringify("true", true, .{}); + try testStringify("null", @as(?u8, null), .{}); + try testStringify("null", @as(?*u32, null), .{}); + try testStringify("42", 42, .{}); + try testStringify("42", 42.0, .{}); + try testStringify("42", @as(u8, 42), .{}); + try testStringify("42", @as(u128, 42), .{}); + try testStringify("9999999999999999", 9999999999999999, .{}); + try testStringify("42", @as(f32, 42), .{}); + try testStringify("42", @as(f64, 42), .{}); + try testStringify("\"ItBroke\"", @as(anyerror, error.ItBroke), .{}); + try testStringify("\"ItBroke\"", error.ItBroke, .{}); +} + +test "stringify string" { + try testStringify("\"hello\"", "hello", .{}); + try testStringify("\"with\\nescapes\\r\"", "with\nescapes\r", .{}); + try testStringify("\"with\\nescapes\\r\"", "with\nescapes\r", .{ .escape_unicode = true }); + try testStringify("\"with unicode\\u0001\"", "with unicode\u{1}", .{}); + try testStringify("\"with unicode\\u0001\"", "with unicode\u{1}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{80}\"", "with unicode\u{80}", .{}); + try testStringify("\"with unicode\\u0080\"", "with unicode\u{80}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{FF}\"", "with unicode\u{FF}", .{}); + try testStringify("\"with unicode\\u00ff\"", "with unicode\u{FF}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{100}\"", "with unicode\u{100}", .{}); + try testStringify("\"with unicode\\u0100\"", "with unicode\u{100}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{800}\"", "with unicode\u{800}", .{}); + try testStringify("\"with unicode\\u0800\"", "with unicode\u{800}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{8000}\"", "with unicode\u{8000}", .{}); + try testStringify("\"with unicode\\u8000\"", "with unicode\u{8000}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{D799}\"", "with unicode\u{D799}", .{}); + try testStringify("\"with unicode\\ud799\"", "with unicode\u{D799}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{10000}\"", "with unicode\u{10000}", .{}); + try testStringify("\"with unicode\\ud800\\udc00\"", "with unicode\u{10000}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{10FFFF}\"", "with unicode\u{10FFFF}", .{}); + try testStringify("\"with unicode\\udbff\\udfff\"", "with unicode\u{10FFFF}", .{ .escape_unicode = true }); +} + +test "stringify many-item sentinel-terminated string" { + try testStringify("\"hello\"", @as([*:0]const u8, "hello"), .{}); + try testStringify("\"with\\nescapes\\r\"", @as([*:0]const u8, "with\nescapes\r"), .{ .escape_unicode = true }); + try testStringify("\"with unicode\\u0001\"", @as([*:0]const u8, "with unicode\u{1}"), .{ .escape_unicode = true }); +} + +test "stringify enums" { + const E = enum { + foo, + bar, + }; + try testStringify("\"foo\"", E.foo, .{}); + try testStringify("\"bar\"", E.bar, .{}); +} + +test "stringify non-exhaustive enum" { + const E = enum(u8) { + foo = 0, + _, + }; + try testStringify("\"foo\"", E.foo, .{}); + try testStringify("1", @as(E, @enumFromInt(1)), .{}); +} + +test "stringify enum literals" { + try testStringify("\"foo\"", .foo, .{}); + try testStringify("\"bar\"", .bar, .{}); +} + +test "stringify tagged unions" { + const T = union(enum) { + nothing, + foo: u32, + bar: bool, + }; + try testStringify("{\"nothing\":{}}", T{ .nothing = {} }, .{}); + try testStringify("{\"foo\":42}", T{ .foo = 42 }, .{}); + try testStringify("{\"bar\":true}", T{ .bar = true }, .{}); +} + +test "stringify struct" { + try testStringify("{\"foo\":42}", struct { + foo: u32, + }{ .foo = 42 }, .{}); +} + +test "emit_strings_as_arrays" { + // Should only affect string values, not object keys. + try testStringify("{\"foo\":\"bar\"}", .{ .foo = "bar" }, .{}); + try testStringify("{\"foo\":[98,97,114]}", .{ .foo = "bar" }, .{ .emit_strings_as_arrays = true }); + // Should *not* affect these types: + try testStringify("\"foo\"", @as(enum { foo, bar }, .foo), .{ .emit_strings_as_arrays = true }); + try testStringify("\"ItBroke\"", error.ItBroke, .{ .emit_strings_as_arrays = true }); + // Should work on these: + try testStringify("\"bar\"", @Vector(3, u8){ 'b', 'a', 'r' }, .{}); + try testStringify("[98,97,114]", @Vector(3, u8){ 'b', 'a', 'r' }, .{ .emit_strings_as_arrays = true }); + try testStringify("\"bar\"", [3]u8{ 'b', 'a', 'r' }, .{}); + try testStringify("[98,97,114]", [3]u8{ 'b', 'a', 'r' }, .{ .emit_strings_as_arrays = true }); +} + +test "stringify struct with indentation" { + try testStringify( + \\{ + \\ "foo": 42, + \\ "bar": [ + \\ 1, + \\ 2, + \\ 3 + \\ ] + \\} + , + struct { + foo: u32, + bar: [3]u32, + }{ + .foo = 42, + .bar = .{ 1, 2, 3 }, + }, + .{ .whitespace = .indent_4 }, + ); + try testStringify( + "{\n\t\"foo\": 42,\n\t\"bar\": [\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}", + struct { + foo: u32, + bar: [3]u32, + }{ + .foo = 42, + .bar = .{ 1, 2, 3 }, + }, + .{ .whitespace = .indent_tab }, + ); + try testStringify( + \\{"foo":42,"bar":[1,2,3]} + , + struct { + foo: u32, + bar: [3]u32, + }{ + .foo = 42, + .bar = .{ 1, 2, 3 }, + }, + .{ .whitespace = .minified }, + ); +} + +test "stringify array of structs" { + const MyStruct = struct { + foo: u32, + }; + try testStringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ + MyStruct{ .foo = 42 }, + MyStruct{ .foo = 100 }, + MyStruct{ .foo = 1000 }, + }, .{}); +} + +test "stringify struct with custom stringifier" { + try testStringify("[\"something special\",42]", struct { + foo: u32, + const Self = @This(); + pub fn jsonStringify(v: @This(), jws: anytype) !void { + _ = v; + try jws.beginArray(); + try jws.write("something special"); + try jws.write(42); + try jws.endArray(); + } + }{ .foo = 42 }, .{}); +} + +fn testStringify(expected: []const u8, v: anytype, options: Options) !void { + var buffer: [4096]u8 = undefined; + var w: Writer = .fixed(&buffer); + try value(v, options, &w); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "raw streaming" { + var out_buf: [1024]u8 = undefined; + var out: Writer = .fixed(&out_buf); + + var w: Stringify = .{ .writer = &out, .options = .{ .whitespace = .indent_2 } }; + try w.beginObject(); + try w.beginObjectFieldRaw(); + try w.writer.writeAll("\"long"); + try w.writer.writeAll(" key\""); + w.endObjectFieldRaw(); + try w.beginWriteRaw(); + try w.writer.writeAll("\"long"); + try w.writer.writeAll(" value\""); + w.endWriteRaw(); + try w.endObject(); + + const expected = + \\{ + \\ "long key": "long value" + \\} + ; + try std.testing.expectEqualStrings(expected, w.writer.buffered()); +} diff --git a/lib/std/json/dynamic.zig b/lib/std/json/dynamic.zig index 4d24444390f7c3ae02b3745b454c2c68c685d0b5..b47e7e1067bbab1bd21664adef7e5a23b2c6424a 100644 --- a/lib/std/json/dynamic.zig +++ b/lib/std/json/dynamic.zig @@ -4,17 +4,12 @@ const ArenaAllocator = std.heap.ArenaAllocator; const ArrayList = std.ArrayList; const StringArrayHashMap = std.StringArrayHashMap; const Allocator = std.mem.Allocator; - -const StringifyOptions = @import("./stringify.zig").StringifyOptions; -const stringify = @import("./stringify.zig").stringify; +const json = std.json; const ParseOptions = @import("./static.zig").ParseOptions; const ParseError = @import("./static.zig").ParseError; -const JsonScanner = @import("./scanner.zig").Scanner; -const AllocWhen = @import("./scanner.zig").AllocWhen; -const Token = @import("./scanner.zig").Token; -const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger; +const isNumberFormattedLikeAnInteger = @import("Scanner.zig").isNumberFormattedLikeAnInteger; pub const ObjectMap = StringArrayHashMap(Value); pub const Array = ArrayList(Value); @@ -52,12 +47,11 @@ pub const Value = union(enum) { } } - pub fn dump(self: Value) void { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); + pub fn dump(v: Value) void { + const w = std.debug.lockStderrWriter(&.{}); + defer std.debug.unlockStderrWriter(); - const stderr = std.fs.File.stderr().deprecatedWriter(); - stringify(self, .{}, stderr) catch return; + json.Stringify.value(v, .{}, w) catch return; } pub fn jsonStringify(value: @This(), jws: anytype) !void { diff --git a/lib/std/json/dynamic_test.zig b/lib/std/json/dynamic_test.zig index 1362e3cfadfb92cdda271127e34d568e2ec683d3..9d991f2c5098db311d3cd812955a8a88acb51b80 100644 --- a/lib/std/json/dynamic_test.zig +++ b/lib/std/json/dynamic_test.zig @@ -1,8 +1,10 @@ const std = @import("std"); +const json = std.json; const mem = std.mem; const testing = std.testing; const ArenaAllocator = std.heap.ArenaAllocator; const Allocator = std.mem.Allocator; +const Writer = std.io.Writer; const ObjectMap = @import("dynamic.zig").ObjectMap; const Array = @import("dynamic.zig").Array; @@ -14,8 +16,7 @@ const parseFromTokenSource = @import("static.zig").parseFromTokenSource; const parseFromValueLeaky = @import("static.zig").parseFromValueLeaky; const ParseOptions = @import("static.zig").ParseOptions; -const jsonReader = @import("scanner.zig").reader; -const JsonReader = @import("scanner.zig").Reader; +const Scanner = @import("Scanner.zig"); test "json.parser.dynamic" { const s = @@ -70,14 +71,10 @@ test "json.parser.dynamic" { try testing.expect(mem.eql(u8, large_int.number_string, "18446744073709551615")); } -const writeStream = @import("./stringify.zig").writeStream; test "write json then parse it" { var out_buffer: [1000]u8 = undefined; - - var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer); - const out_stream = fixed_buffer_stream.writer(); - var jw = writeStream(out_stream, .{}); - defer jw.deinit(); + var fixed_writer: Writer = .fixed(&out_buffer); + var jw: json.Stringify = .{ .writer = &fixed_writer, .options = .{} }; try jw.beginObject(); @@ -101,8 +98,8 @@ test "write json then parse it" { try jw.endObject(); - fixed_buffer_stream = std.io.fixedBufferStream(fixed_buffer_stream.getWritten()); - var json_reader = jsonReader(testing.allocator, fixed_buffer_stream.reader()); + var fbs: std.Io.Reader = .fixed(fixed_writer.buffered()); + var json_reader: Scanner.Reader = .init(testing.allocator, &fbs); defer json_reader.deinit(); var parsed = try parseFromTokenSource(Value, testing.allocator, &json_reader, .{}); defer parsed.deinit(); @@ -242,10 +239,9 @@ test "Value.jsonStringify" { .{ .object = obj }, }; var buffer: [0x1000]u8 = undefined; - var fbs = std.io.fixedBufferStream(&buffer); + var fixed_writer: Writer = .fixed(&buffer); - var jw = writeStream(fbs.writer(), .{ .whitespace = .indent_1 }); - defer jw.deinit(); + var jw: json.Stringify = .{ .writer = &fixed_writer, .options = .{ .whitespace = .indent_1 } }; try jw.write(array); const expected = @@ -266,7 +262,7 @@ test "Value.jsonStringify" { \\ } \\] ; - try testing.expectEqualStrings(expected, fbs.getWritten()); + try testing.expectEqualStrings(expected, fixed_writer.buffered()); } test "parseFromValue(std.json.Value,...)" { @@ -334,8 +330,8 @@ test "polymorphic parsing" { test "long object value" { const value = "01234567890123456789"; const doc = "{\"key\":\"" ++ value ++ "\"}"; - var fbs = std.io.fixedBufferStream(doc); - var reader = smallBufferJsonReader(testing.allocator, fbs.reader()); + var fbs: std.Io.Reader = .fixed(doc); + var reader = smallBufferJsonReader(testing.allocator, &fbs); defer reader.deinit(); var parsed = try parseFromTokenSource(Value, testing.allocator, &reader, .{}); defer parsed.deinit(); @@ -367,8 +363,8 @@ test "many object keys" { \\ "k5": "v5" \\} ; - var fbs = std.io.fixedBufferStream(doc); - var reader = smallBufferJsonReader(testing.allocator, fbs.reader()); + var fbs: std.Io.Reader = .fixed(doc); + var reader = smallBufferJsonReader(testing.allocator, &fbs); defer reader.deinit(); var parsed = try parseFromTokenSource(Value, testing.allocator, &reader, .{}); defer parsed.deinit(); @@ -382,8 +378,8 @@ test "many object keys" { test "negative zero" { const doc = "-0"; - var fbs = std.io.fixedBufferStream(doc); - var reader = smallBufferJsonReader(testing.allocator, fbs.reader()); + var fbs: std.Io.Reader = .fixed(doc); + var reader = smallBufferJsonReader(testing.allocator, &fbs); defer reader.deinit(); var parsed = try parseFromTokenSource(Value, testing.allocator, &reader, .{}); defer parsed.deinit(); @@ -391,6 +387,6 @@ test "negative zero" { try testing.expect(std.math.isNegativeZero(parsed.value.float)); } -fn smallBufferJsonReader(allocator: Allocator, io_reader: anytype) JsonReader(16, @TypeOf(io_reader)) { - return JsonReader(16, @TypeOf(io_reader)).init(allocator, io_reader); +fn smallBufferJsonReader(allocator: Allocator, io_reader: anytype) Scanner.Reader { + return .init(allocator, io_reader); } diff --git a/lib/std/json/fmt.zig b/lib/std/json/fmt.zig deleted file mode 100644 index c526dce1da8546ab8c3f4c99944dd2329dd74478..0000000000000000000000000000000000000000 --- a/lib/std/json/fmt.zig +++ /dev/null @@ -1,40 +0,0 @@ -const std = @import("../std.zig"); -const assert = std.debug.assert; - -const stringify = @import("stringify.zig").stringify; -const StringifyOptions = @import("stringify.zig").StringifyOptions; - -/// Returns a formatter that formats the given value using stringify. -pub fn fmt(value: anytype, options: StringifyOptions) Formatter(@TypeOf(value)) { - return Formatter(@TypeOf(value)){ .value = value, .options = options }; -} - -/// Formats the given value using stringify. -pub fn Formatter(comptime T: type) type { - return struct { - value: T, - options: StringifyOptions, - - pub fn format(self: @This(), writer: *std.io.Writer) std.io.Writer.Error!void { - try stringify(self.value, self.options, writer); - } - }; -} - -test fmt { - const expectFmt = std.testing.expectFmt; - try expectFmt("123", "{}", .{fmt(@as(u32, 123), .{})}); - try expectFmt( - \\{"num":927,"msg":"hello","sub":{"mybool":true}} - , "{}", .{fmt(struct { - num: u32, - msg: []const u8, - sub: struct { - mybool: bool, - }, - }{ - .num = 927, - .msg = "hello", - .sub = .{ .mybool = true }, - }, .{})}); -} diff --git a/lib/std/json/hashmap_test.zig b/lib/std/json/hashmap_test.zig index 49d8caffae70f6a263b09375ab4e83ed1f926929..0544eaa68ba66f240b48726212b84d9f3346178c 100644 --- a/lib/std/json/hashmap_test.zig +++ b/lib/std/json/hashmap_test.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const json = std.json; const testing = std.testing; const ArrayHashMap = @import("hashmap.zig").ArrayHashMap; @@ -7,10 +8,9 @@ const parseFromSlice = @import("static.zig").parseFromSlice; const parseFromSliceLeaky = @import("static.zig").parseFromSliceLeaky; const parseFromTokenSource = @import("static.zig").parseFromTokenSource; const parseFromValue = @import("static.zig").parseFromValue; -const stringifyAlloc = @import("stringify.zig").stringifyAlloc; const Value = @import("dynamic.zig").Value; -const jsonReader = @import("./scanner.zig").reader; +const Scanner = @import("Scanner.zig"); const T = struct { i: i32, @@ -39,8 +39,8 @@ test "parse json hashmap while streaming" { \\ "xyz": {"i": 1, "s": "w"} \\} ; - var stream = std.io.fixedBufferStream(doc); - var json_reader = jsonReader(testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(doc); + var json_reader: Scanner.Reader = .init(testing.allocator, &stream); var parsed = try parseFromTokenSource( ArrayHashMap(T), @@ -89,7 +89,7 @@ test "stringify json hashmap" { var value = ArrayHashMap(T){}; defer value.deinit(testing.allocator); { - const doc = try stringifyAlloc(testing.allocator, value, .{}); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{}); defer testing.allocator.free(doc); try testing.expectEqualStrings("{}", doc); } @@ -98,7 +98,7 @@ test "stringify json hashmap" { try value.map.put(testing.allocator, "xyz", .{ .i = 1, .s = "w" }); { - const doc = try stringifyAlloc(testing.allocator, value, .{}); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{}); defer testing.allocator.free(doc); try testing.expectEqualStrings( \\{"abc":{"i":0,"s":"d"},"xyz":{"i":1,"s":"w"}} @@ -107,7 +107,7 @@ test "stringify json hashmap" { try testing.expect(value.map.swapRemove("abc")); { - const doc = try stringifyAlloc(testing.allocator, value, .{}); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{}); defer testing.allocator.free(doc); try testing.expectEqualStrings( \\{"xyz":{"i":1,"s":"w"}} @@ -116,7 +116,7 @@ test "stringify json hashmap" { try testing.expect(value.map.swapRemove("xyz")); { - const doc = try stringifyAlloc(testing.allocator, value, .{}); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{}); defer testing.allocator.free(doc); try testing.expectEqualStrings("{}", doc); } @@ -129,7 +129,7 @@ test "stringify json hashmap whitespace" { try value.map.put(testing.allocator, "xyz", .{ .i = 1, .s = "w" }); { - const doc = try stringifyAlloc(testing.allocator, value, .{ .whitespace = .indent_2 }); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{ .whitespace = .indent_2 }); defer testing.allocator.free(doc); try testing.expectEqualStrings( \\{ diff --git a/lib/std/json/scanner.zig b/lib/std/json/scanner.zig deleted file mode 100644 index 1836d6775bec3c4115240d166c92561848da918c..0000000000000000000000000000000000000000 --- a/lib/std/json/scanner.zig +++ /dev/null @@ -1,1776 +0,0 @@ -// Notes on standards compliance: https://datatracker.ietf.org/doc/html/rfc8259 -// * RFC 8259 requires JSON documents be valid UTF-8, -// but makes an allowance for systems that are "part of a closed ecosystem". -// I have no idea what that's supposed to mean in the context of a standard specification. -// This implementation requires inputs to be valid UTF-8. -// * RFC 8259 contradicts itself regarding whether lowercase is allowed in \u hex digits, -// but this is probably a bug in the spec, and it's clear that lowercase is meant to be allowed. -// (RFC 5234 defines HEXDIG to only allow uppercase.) -// * When RFC 8259 refers to a "character", I assume they really mean a "Unicode scalar value". -// See http://www.unicode.org/glossary/#unicode_scalar_value . -// * RFC 8259 doesn't explicitly disallow unpaired surrogate halves in \u escape sequences, -// but vaguely implies that \u escapes are for encoding Unicode "characters" (i.e. Unicode scalar values?), -// which would mean that unpaired surrogate halves are forbidden. -// By contrast ECMA-404 (a competing(/compatible?) JSON standard, which JavaScript's JSON.parse() conforms to) -// explicitly allows unpaired surrogate halves. -// This implementation forbids unpaired surrogate halves in \u sequences. -// If a high surrogate half appears in a \u sequence, -// then a low surrogate half must immediately follow in \u notation. -// * RFC 8259 allows implementations to "accept non-JSON forms or extensions". -// This implementation does not accept any of that. -// * RFC 8259 allows implementations to put limits on "the size of texts", -// "the maximum depth of nesting", "the range and precision of numbers", -// and "the length and character contents of strings". -// This low-level implementation does not limit these, -// except where noted above, and except that nesting depth requires memory allocation. -// Note that this low-level API does not interpret numbers numerically, -// but simply emits their source form for some higher level code to make sense of. -// * This low-level implementation allows duplicate object keys, -// and key/value pairs are emitted in the order they appear in the input. - -const std = @import("std"); - -const Allocator = std.mem.Allocator; -const ArrayList = std.ArrayList; -const assert = std.debug.assert; -const BitStack = std.BitStack; - -/// Scan the input and check for malformed JSON. -/// On `SyntaxError` or `UnexpectedEndOfInput`, returns `false`. -/// Returns any errors from the allocator as-is, which is unlikely, -/// but can be caused by extreme nesting depth in the input. -pub fn validate(allocator: Allocator, s: []const u8) Allocator.Error!bool { - var scanner = Scanner.initCompleteInput(allocator, s); - defer scanner.deinit(); - - while (true) { - const token = scanner.next() catch |err| switch (err) { - error.SyntaxError, error.UnexpectedEndOfInput => return false, - error.OutOfMemory => return error.OutOfMemory, - error.BufferUnderrun => unreachable, - }; - if (token == .end_of_document) break; - } - - return true; -} - -/// The parsing errors are divided into two categories: -/// * `SyntaxError` is for clearly malformed JSON documents, -/// such as giving an input document that isn't JSON at all. -/// * `UnexpectedEndOfInput` is for signaling that everything's been -/// valid so far, but the input appears to be truncated for some reason. -/// Note that a completely empty (or whitespace-only) input will give `UnexpectedEndOfInput`. -pub const Error = error{ SyntaxError, UnexpectedEndOfInput }; - -/// Calls `std.json.Reader` with `std.json.default_buffer_size`. -pub fn reader(allocator: Allocator, io_reader: anytype) Reader(default_buffer_size, @TypeOf(io_reader)) { - return Reader(default_buffer_size, @TypeOf(io_reader)).init(allocator, io_reader); -} -/// Used by `json.reader`. -pub const default_buffer_size = 0x1000; - -/// The tokens emitted by `std.json.Scanner` and `std.json.Reader` `.next*()` functions follow this grammar: -/// ``` -/// = .end_of_document -/// = -/// | -/// | -/// | -/// | -/// | .true -/// | .false -/// | .null -/// = .object_begin ( )* .object_end -/// = .array_begin ( )* .array_end -/// = -/// = -/// ``` -/// -/// What you get for `` and `` values depends on which `next*()` method you call: -/// -/// ``` -/// next(): -/// = ( .partial_number )* .number -/// = ( )* .string -/// = -/// | .partial_string -/// | .partial_string_escaped_1 -/// | .partial_string_escaped_2 -/// | .partial_string_escaped_3 -/// | .partial_string_escaped_4 -/// -/// nextAlloc*(..., .alloc_always): -/// = .allocated_number -/// = .allocated_string -/// -/// nextAlloc*(..., .alloc_if_needed): -/// = -/// | .number -/// | .allocated_number -/// = -/// | .string -/// | .allocated_string -/// ``` -/// -/// For all tokens with a `[]const u8`, `[]u8`, or `[n]u8` payload, the payload represents the content of the value. -/// For number values, this is the representation of the number exactly as it appears in the input. -/// For strings, this is the content of the string after resolving escape sequences. -/// -/// For `.allocated_number` and `.allocated_string`, the `[]u8` payloads are allocations made with the given allocator. -/// You are responsible for managing that memory. `json.Reader.deinit()` does *not* free those allocations. -/// -/// The `.partial_*` tokens indicate that a value spans multiple input buffers or that a string contains escape sequences. -/// To get a complete value in memory, you need to concatenate the values yourself. -/// Calling `nextAlloc*()` does this for you, and returns an `.allocated_*` token with the result. -/// -/// For tokens with a `[]const u8` payload, the payload is a slice into the current input buffer. -/// The memory may become undefined during the next call to `json.Scanner.feedInput()` -/// or any `json.Reader` method whose return error set includes `json.Error`. -/// To keep the value persistently, it recommended to make a copy or to use `.alloc_always`, -/// which makes a copy for you. -/// -/// Note that `.number` and `.string` tokens that follow `.partial_*` tokens may have `0` length to indicate that -/// the previously partial value is completed with no additional bytes. -/// (This can happen when the break between input buffers happens to land on the exact end of a value. E.g. `"[1234"`, `"]"`.) -/// `.partial_*` tokens never have `0` length. -/// -/// The recommended strategy for using the different `next*()` methods is something like this: -/// -/// When you're expecting an object key, use `.alloc_if_needed`. -/// You often don't need a copy of the key string to persist; you might just check which field it is. -/// In the case that the key happens to require an allocation, free it immediately after checking it. -/// -/// When you're expecting a meaningful string value (such as on the right of a `:`), -/// use `.alloc_always` in order to keep the value valid throughout parsing the rest of the document. -/// -/// When you're expecting a number value, use `.alloc_if_needed`. -/// You're probably going to be parsing the string representation of the number into a numeric representation, -/// so you need the complete string representation only temporarily. -/// -/// When you're skipping an unrecognized value, use `skipValue()`. -pub const Token = union(enum) { - object_begin, - object_end, - array_begin, - array_end, - - true, - false, - null, - - number: []const u8, - partial_number: []const u8, - allocated_number: []u8, - - string: []const u8, - partial_string: []const u8, - partial_string_escaped_1: [1]u8, - partial_string_escaped_2: [2]u8, - partial_string_escaped_3: [3]u8, - partial_string_escaped_4: [4]u8, - allocated_string: []u8, - - end_of_document, -}; - -/// This is only used in `peekNextTokenType()` and gives a categorization based on the first byte of the next token that will be emitted from a `next*()` call. -pub const TokenType = enum { - object_begin, - object_end, - array_begin, - array_end, - true, - false, - null, - number, - string, - end_of_document, -}; - -/// To enable diagnostics, declare `var diagnostics = Diagnostics{};` then call `source.enableDiagnostics(&diagnostics);` -/// where `source` is either a `std.json.Reader` or a `std.json.Scanner` that has just been initialized. -/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()` -/// to get meaningful information from this. -pub const Diagnostics = struct { - line_number: u64 = 1, - line_start_cursor: usize = @as(usize, @bitCast(@as(isize, -1))), // Start just "before" the input buffer to get a 1-based column for line 1. - total_bytes_before_current_input: u64 = 0, - cursor_pointer: *const usize = undefined, - - /// Starts at 1. - pub fn getLine(self: *const @This()) u64 { - return self.line_number; - } - /// Starts at 1. - pub fn getColumn(self: *const @This()) u64 { - return self.cursor_pointer.* -% self.line_start_cursor; - } - /// Starts at 0. Measures the byte offset since the start of the input. - pub fn getByteOffset(self: *const @This()) u64 { - return self.total_bytes_before_current_input + self.cursor_pointer.*; - } -}; - -/// See the documentation for `std.json.Token`. -pub const AllocWhen = enum { alloc_if_needed, alloc_always }; - -/// For security, the maximum size allocated to store a single string or number value is limited to 4MiB by default. -/// This limit can be specified by calling `nextAllocMax()` instead of `nextAlloc()`. -pub const default_max_value_len = 4 * 1024 * 1024; - -/// Connects a `std.io.GenericReader` to a `std.json.Scanner`. -/// All `next*()` methods here handle `error.BufferUnderrun` from `std.json.Scanner`, and then read from the reader. -pub fn Reader(comptime buffer_size: usize, comptime ReaderType: type) type { - return struct { - scanner: Scanner, - reader: ReaderType, - - buffer: [buffer_size]u8 = undefined, - - /// The allocator is only used to track `[]` and `{}` nesting levels. - pub fn init(allocator: Allocator, io_reader: ReaderType) @This() { - return .{ - .scanner = Scanner.initStreaming(allocator), - .reader = io_reader, - }; - } - pub fn deinit(self: *@This()) void { - self.scanner.deinit(); - self.* = undefined; - } - - /// Calls `std.json.Scanner.enableDiagnostics`. - pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void { - self.scanner.enableDiagnostics(diagnostics); - } - - pub const NextError = ReaderType.Error || Error || Allocator.Error; - pub const SkipError = NextError; - pub const AllocError = NextError || error{ValueTooLong}; - pub const PeekError = ReaderType.Error || Error; - - /// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);` - /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. - pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token { - return self.nextAllocMax(allocator, when, default_max_value_len); - } - /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. - pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token { - const token_type = try self.peekNextTokenType(); - switch (token_type) { - .number, .string => { - var value_list = ArrayList(u8).init(allocator); - errdefer { - value_list.deinit(); - } - if (try self.allocNextIntoArrayListMax(&value_list, when, max_value_len)) |slice| { - return if (token_type == .number) - Token{ .number = slice } - else - Token{ .string = slice }; - } else { - return if (token_type == .number) - Token{ .allocated_number = try value_list.toOwnedSlice() } - else - Token{ .allocated_string = try value_list.toOwnedSlice() }; - } - }, - - // Simple tokens never alloc. - .object_begin, - .object_end, - .array_begin, - .array_end, - .true, - .false, - .null, - .end_of_document, - => return try self.next(), - } - } - - /// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);` - pub fn allocNextIntoArrayList(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen) AllocError!?[]const u8 { - return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len); - } - /// Calls `std.json.Scanner.allocNextIntoArrayListMax` and handles `error.BufferUnderrun`. - pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocError!?[]const u8 { - while (true) { - return self.scanner.allocNextIntoArrayListMax(value_list, when, max_value_len) catch |err| switch (err) { - error.BufferUnderrun => { - try self.refillBuffer(); - continue; - }, - else => |other_err| return other_err, - }; - } - } - - /// Like `std.json.Scanner.skipValue`, but handles `error.BufferUnderrun`. - pub fn skipValue(self: *@This()) SkipError!void { - switch (try self.peekNextTokenType()) { - .object_begin, .array_begin => { - try self.skipUntilStackHeight(self.stackHeight()); - }, - .number, .string => { - while (true) { - switch (try self.next()) { - .partial_number, - .partial_string, - .partial_string_escaped_1, - .partial_string_escaped_2, - .partial_string_escaped_3, - .partial_string_escaped_4, - => continue, - - .number, .string => break, - - else => unreachable, - } - } - }, - .true, .false, .null => { - _ = try self.next(); - }, - - .object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token. - } - } - /// Like `std.json.Scanner.skipUntilStackHeight()` but handles `error.BufferUnderrun`. - pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void { - while (true) { - return self.scanner.skipUntilStackHeight(terminal_stack_height) catch |err| switch (err) { - error.BufferUnderrun => { - try self.refillBuffer(); - continue; - }, - else => |other_err| return other_err, - }; - } - } - - /// Calls `std.json.Scanner.stackHeight`. - pub fn stackHeight(self: *const @This()) usize { - return self.scanner.stackHeight(); - } - /// Calls `std.json.Scanner.ensureTotalStackCapacity`. - pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void { - try self.scanner.ensureTotalStackCapacity(height); - } - - /// See `std.json.Token` for documentation of this function. - pub fn next(self: *@This()) NextError!Token { - while (true) { - return self.scanner.next() catch |err| switch (err) { - error.BufferUnderrun => { - try self.refillBuffer(); - continue; - }, - else => |other_err| return other_err, - }; - } - } - - /// See `std.json.Scanner.peekNextTokenType()`. - pub fn peekNextTokenType(self: *@This()) PeekError!TokenType { - while (true) { - return self.scanner.peekNextTokenType() catch |err| switch (err) { - error.BufferUnderrun => { - try self.refillBuffer(); - continue; - }, - else => |other_err| return other_err, - }; - } - } - - fn refillBuffer(self: *@This()) ReaderType.Error!void { - const input = self.buffer[0..try self.reader.read(self.buffer[0..])]; - if (input.len > 0) { - self.scanner.feedInput(input); - } else { - self.scanner.endInput(); - } - } - }; -} - -/// The lowest level parsing API in this package; -/// supports streaming input with a low memory footprint. -/// The memory requirement is `O(d)` where d is the nesting depth of `[]` or `{}` containers in the input. -/// Specifically `d/8` bytes are required for this purpose, -/// with some extra buffer according to the implementation of `std.ArrayList`. -/// -/// This scanner can emit partial tokens; see `std.json.Token`. -/// The input to this class is a sequence of input buffers that you must supply one at a time. -/// Call `feedInput()` with the first buffer, then call `next()` repeatedly until `error.BufferUnderrun` is returned. -/// Then call `feedInput()` again and so forth. -/// Call `endInput()` when the last input buffer has been given to `feedInput()`, either immediately after calling `feedInput()`, -/// or when `error.BufferUnderrun` requests more data and there is no more. -/// Be sure to call `next()` after calling `endInput()` until `Token.end_of_document` has been returned. -pub const Scanner = struct { - state: State = .value, - string_is_object_key: bool = false, - stack: BitStack, - value_start: usize = undefined, - utf16_code_units: [2]u16 = undefined, - - input: []const u8 = "", - cursor: usize = 0, - is_end_of_input: bool = false, - diagnostics: ?*Diagnostics = null, - - /// The allocator is only used to track `[]` and `{}` nesting levels. - pub fn initStreaming(allocator: Allocator) @This() { - return .{ - .stack = BitStack.init(allocator), - }; - } - /// Use this if your input is a single slice. - /// This is effectively equivalent to: - /// ``` - /// initStreaming(allocator); - /// feedInput(complete_input); - /// endInput(); - /// ``` - pub fn initCompleteInput(allocator: Allocator, complete_input: []const u8) @This() { - return .{ - .stack = BitStack.init(allocator), - .input = complete_input, - .is_end_of_input = true, - }; - } - pub fn deinit(self: *@This()) void { - self.stack.deinit(); - self.* = undefined; - } - - pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void { - diagnostics.cursor_pointer = &self.cursor; - self.diagnostics = diagnostics; - } - - /// Call this whenever you get `error.BufferUnderrun` from `next()`. - /// When there is no more input to provide, call `endInput()`. - pub fn feedInput(self: *@This(), input: []const u8) void { - assert(self.cursor == self.input.len); // Not done with the last input slice. - if (self.diagnostics) |diag| { - diag.total_bytes_before_current_input += self.input.len; - // This usually goes "negative" to measure how far before the beginning - // of the new buffer the current line started. - diag.line_start_cursor -%= self.cursor; - } - self.input = input; - self.cursor = 0; - self.value_start = 0; - } - /// Call this when you will no longer call `feedInput()` anymore. - /// This can be called either immediately after the last `feedInput()`, - /// or at any time afterward, such as when getting `error.BufferUnderrun` from `next()`. - /// Don't forget to call `next*()` after `endInput()` until you get `.end_of_document`. - pub fn endInput(self: *@This()) void { - self.is_end_of_input = true; - } - - pub const NextError = Error || Allocator.Error || error{BufferUnderrun}; - pub const AllocError = Error || Allocator.Error || error{ValueTooLong}; - pub const PeekError = Error || error{BufferUnderrun}; - pub const SkipError = Error || Allocator.Error; - pub const AllocIntoArrayListError = AllocError || error{BufferUnderrun}; - - /// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);` - /// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. - /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. - pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token { - return self.nextAllocMax(allocator, when, default_max_value_len); - } - - /// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. - /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. - pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token { - assert(self.is_end_of_input); // This function is not available in streaming mode. - const token_type = self.peekNextTokenType() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }; - switch (token_type) { - .number, .string => { - var value_list = ArrayList(u8).init(allocator); - errdefer { - value_list.deinit(); - } - if (self.allocNextIntoArrayListMax(&value_list, when, max_value_len) catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }) |slice| { - return if (token_type == .number) - Token{ .number = slice } - else - Token{ .string = slice }; - } else { - return if (token_type == .number) - Token{ .allocated_number = try value_list.toOwnedSlice() } - else - Token{ .allocated_string = try value_list.toOwnedSlice() }; - } - }, - - // Simple tokens never alloc. - .object_begin, - .object_end, - .array_begin, - .array_end, - .true, - .false, - .null, - .end_of_document, - => return self.next() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }, - } - } - - /// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);` - pub fn allocNextIntoArrayList(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen) AllocIntoArrayListError!?[]const u8 { - return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len); - } - /// The next token type must be either `.number` or `.string`. See `peekNextTokenType()`. - /// When allocation is not necessary with `.alloc_if_needed`, - /// this method returns the content slice from the input buffer, and `value_list` is not touched. - /// When allocation is necessary or with `.alloc_always`, this method concatenates partial tokens into the given `value_list`, - /// and returns `null` once the final `.number` or `.string` token has been written into it. - /// In case of an `error.BufferUnderrun`, partial values will be left in the given value_list. - /// The given `value_list` is never reset by this method, so an `error.BufferUnderrun` situation - /// can be resumed by passing the same array list in again. - /// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type; - /// the caller of this method is expected to know which type of token is being processed. - pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 { - while (true) { - const token = try self.next(); - switch (token) { - // Accumulate partial values. - .partial_number, .partial_string => |slice| { - try appendSlice(value_list, slice, max_value_len); - }, - .partial_string_escaped_1 => |buf| { - try appendSlice(value_list, buf[0..], max_value_len); - }, - .partial_string_escaped_2 => |buf| { - try appendSlice(value_list, buf[0..], max_value_len); - }, - .partial_string_escaped_3 => |buf| { - try appendSlice(value_list, buf[0..], max_value_len); - }, - .partial_string_escaped_4 => |buf| { - try appendSlice(value_list, buf[0..], max_value_len); - }, - - // Return complete values. - .number => |slice| { - if (when == .alloc_if_needed and value_list.items.len == 0) { - // No alloc necessary. - return slice; - } - try appendSlice(value_list, slice, max_value_len); - // The token is complete. - return null; - }, - .string => |slice| { - if (when == .alloc_if_needed and value_list.items.len == 0) { - // No alloc necessary. - return slice; - } - try appendSlice(value_list, slice, max_value_len); - // The token is complete. - return null; - }, - - .object_begin, - .object_end, - .array_begin, - .array_end, - .true, - .false, - .null, - .end_of_document, - => unreachable, // Only .number and .string token types are allowed here. Check peekNextTokenType() before calling this. - - .allocated_number, .allocated_string => unreachable, - } - } - } - - /// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. - /// If the next token type is `.object_begin` or `.array_begin`, - /// this function calls `next()` repeatedly until the corresponding `.object_end` or `.array_end` is found. - /// If the next token type is `.number` or `.string`, - /// this function calls `next()` repeatedly until the (non `.partial_*`) `.number` or `.string` token is found. - /// If the next token type is `.true`, `.false`, or `.null`, this function calls `next()` once. - /// The next token type must not be `.object_end`, `.array_end`, or `.end_of_document`; - /// see `peekNextTokenType()`. - pub fn skipValue(self: *@This()) SkipError!void { - assert(self.is_end_of_input); // This function is not available in streaming mode. - switch (self.peekNextTokenType() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }) { - .object_begin, .array_begin => { - self.skipUntilStackHeight(self.stackHeight()) catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }; - }, - .number, .string => { - while (true) { - switch (self.next() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }) { - .partial_number, - .partial_string, - .partial_string_escaped_1, - .partial_string_escaped_2, - .partial_string_escaped_3, - .partial_string_escaped_4, - => continue, - - .number, .string => break, - - else => unreachable, - } - } - }, - .true, .false, .null => { - _ = self.next() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }; - }, - - .object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token. - } - } - - /// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height. - /// Unlike `skipValue()`, this function is available in streaming mode. - pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void { - while (true) { - switch (try self.next()) { - .object_end, .array_end => { - if (self.stackHeight() == terminal_stack_height) break; - }, - .end_of_document => unreachable, - else => continue, - } - } - } - - /// The depth of `{}` or `[]` nesting levels at the current position. - pub fn stackHeight(self: *const @This()) usize { - return self.stack.bit_len; - } - - /// Pre allocate memory to hold the given number of nesting levels. - /// `stackHeight()` up to the given number will not cause allocations. - pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void { - try self.stack.ensureTotalCapacity(height); - } - - /// See `std.json.Token` for documentation of this function. - pub fn next(self: *@This()) NextError!Token { - state_loop: while (true) { - switch (self.state) { - .value => { - switch (try self.skipWhitespaceExpectByte()) { - // Object, Array - '{' => { - try self.stack.push(OBJECT_MODE); - self.cursor += 1; - self.state = .object_start; - return .object_begin; - }, - '[' => { - try self.stack.push(ARRAY_MODE); - self.cursor += 1; - self.state = .array_start; - return .array_begin; - }, - - // String - '"' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - continue :state_loop; - }, - - // Number - '1'...'9' => { - self.value_start = self.cursor; - self.cursor += 1; - self.state = .number_int; - continue :state_loop; - }, - '0' => { - self.value_start = self.cursor; - self.cursor += 1; - self.state = .number_leading_zero; - continue :state_loop; - }, - '-' => { - self.value_start = self.cursor; - self.cursor += 1; - self.state = .number_minus; - continue :state_loop; - }, - - // literal values - 't' => { - self.cursor += 1; - self.state = .literal_t; - continue :state_loop; - }, - 'f' => { - self.cursor += 1; - self.state = .literal_f; - continue :state_loop; - }, - 'n' => { - self.cursor += 1; - self.state = .literal_n; - continue :state_loop; - }, - - else => return error.SyntaxError, - } - }, - - .post_value => { - if (try self.skipWhitespaceCheckEnd()) return .end_of_document; - - const c = self.input[self.cursor]; - if (self.string_is_object_key) { - self.string_is_object_key = false; - switch (c) { - ':' => { - self.cursor += 1; - self.state = .value; - continue :state_loop; - }, - else => return error.SyntaxError, - } - } - - switch (c) { - '}' => { - if (self.stack.pop() != OBJECT_MODE) return error.SyntaxError; - self.cursor += 1; - // stay in .post_value state. - return .object_end; - }, - ']' => { - if (self.stack.pop() != ARRAY_MODE) return error.SyntaxError; - self.cursor += 1; - // stay in .post_value state. - return .array_end; - }, - ',' => { - switch (self.stack.peek()) { - OBJECT_MODE => { - self.state = .object_post_comma; - }, - ARRAY_MODE => { - self.state = .value; - }, - } - self.cursor += 1; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - - .object_start => { - switch (try self.skipWhitespaceExpectByte()) { - '"' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - self.string_is_object_key = true; - continue :state_loop; - }, - '}' => { - self.cursor += 1; - _ = self.stack.pop(); - self.state = .post_value; - return .object_end; - }, - else => return error.SyntaxError, - } - }, - .object_post_comma => { - switch (try self.skipWhitespaceExpectByte()) { - '"' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - self.string_is_object_key = true; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - - .array_start => { - switch (try self.skipWhitespaceExpectByte()) { - ']' => { - self.cursor += 1; - _ = self.stack.pop(); - self.state = .post_value; - return .array_end; - }, - else => { - self.state = .value; - continue :state_loop; - }, - } - }, - - .number_minus => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); - switch (self.input[self.cursor]) { - '0' => { - self.cursor += 1; - self.state = .number_leading_zero; - continue :state_loop; - }, - '1'...'9' => { - self.cursor += 1; - self.state = .number_int; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .number_leading_zero => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(true); - switch (self.input[self.cursor]) { - '.' => { - self.cursor += 1; - self.state = .number_post_dot; - continue :state_loop; - }, - 'e', 'E' => { - self.cursor += 1; - self.state = .number_post_e; - continue :state_loop; - }, - else => { - self.state = .post_value; - return Token{ .number = self.takeValueSlice() }; - }, - } - }, - .number_int => { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - '0'...'9' => continue, - '.' => { - self.cursor += 1; - self.state = .number_post_dot; - continue :state_loop; - }, - 'e', 'E' => { - self.cursor += 1; - self.state = .number_post_e; - continue :state_loop; - }, - else => { - self.state = .post_value; - return Token{ .number = self.takeValueSlice() }; - }, - } - } - return self.endOfBufferInNumber(true); - }, - .number_post_dot => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); - switch (self.input[self.cursor]) { - '0'...'9' => { - self.cursor += 1; - self.state = .number_frac; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .number_frac => { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - '0'...'9' => continue, - 'e', 'E' => { - self.cursor += 1; - self.state = .number_post_e; - continue :state_loop; - }, - else => { - self.state = .post_value; - return Token{ .number = self.takeValueSlice() }; - }, - } - } - return self.endOfBufferInNumber(true); - }, - .number_post_e => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); - switch (self.input[self.cursor]) { - '0'...'9' => { - self.cursor += 1; - self.state = .number_exp; - continue :state_loop; - }, - '+', '-' => { - self.cursor += 1; - self.state = .number_post_e_sign; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .number_post_e_sign => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); - switch (self.input[self.cursor]) { - '0'...'9' => { - self.cursor += 1; - self.state = .number_exp; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .number_exp => { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - '0'...'9' => continue, - else => { - self.state = .post_value; - return Token{ .number = self.takeValueSlice() }; - }, - } - } - return self.endOfBufferInNumber(true); - }, - - .string => { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - 0...0x1f => return error.SyntaxError, // Bare ASCII control code in string. - - // ASCII plain text. - 0x20...('"' - 1), ('"' + 1)...('\\' - 1), ('\\' + 1)...0x7F => continue, - - // Special characters. - '"' => { - const result = Token{ .string = self.takeValueSlice() }; - self.cursor += 1; - self.state = .post_value; - return result; - }, - '\\' => { - const slice = self.takeValueSlice(); - self.cursor += 1; - self.state = .string_backslash; - if (slice.len > 0) return Token{ .partial_string = slice }; - continue :state_loop; - }, - - // UTF-8 validation. - // See http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String - 0xC2...0xDF => { - self.cursor += 1; - self.state = .string_utf8_last_byte; - continue :state_loop; - }, - 0xE0 => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte_guard_against_overlong; - continue :state_loop; - }, - 0xE1...0xEC, 0xEE...0xEF => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte; - continue :state_loop; - }, - 0xED => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte_guard_against_surrogate_half; - continue :state_loop; - }, - 0xF0 => { - self.cursor += 1; - self.state = .string_utf8_third_to_last_byte_guard_against_overlong; - continue :state_loop; - }, - 0xF1...0xF3 => { - self.cursor += 1; - self.state = .string_utf8_third_to_last_byte; - continue :state_loop; - }, - 0xF4 => { - self.cursor += 1; - self.state = .string_utf8_third_to_last_byte_guard_against_too_large; - continue :state_loop; - }, - 0x80...0xC1, 0xF5...0xFF => return error.SyntaxError, // Invalid UTF-8. - } - } - if (self.is_end_of_input) return error.UnexpectedEndOfInput; - const slice = self.takeValueSlice(); - if (slice.len > 0) return Token{ .partial_string = slice }; - return error.BufferUnderrun; - }, - .string_backslash => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - '"', '\\', '/' => { - // Since these characters now represent themselves literally, - // we can simply begin the next plaintext slice here. - self.value_start = self.cursor; - self.cursor += 1; - self.state = .string; - continue :state_loop; - }, - 'b' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{0x08} }; - }, - 'f' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{0x0c} }; - }, - 'n' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{'\n'} }; - }, - 'r' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{'\r'} }; - }, - 't' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{'\t'} }; - }, - 'u' => { - self.cursor += 1; - self.state = .string_backslash_u; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .string_backslash_u => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[0] = @as(u16, c - '0') << 12; - }, - 'A'...'F' => { - self.utf16_code_units[0] = @as(u16, c - 'A' + 10) << 12; - }, - 'a'...'f' => { - self.utf16_code_units[0] = @as(u16, c - 'a' + 10) << 12; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - self.state = .string_backslash_u_1; - continue :state_loop; - }, - .string_backslash_u_1 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[0] |= @as(u16, c - '0') << 8; - }, - 'A'...'F' => { - self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 8; - }, - 'a'...'f' => { - self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 8; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - self.state = .string_backslash_u_2; - continue :state_loop; - }, - .string_backslash_u_2 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[0] |= @as(u16, c - '0') << 4; - }, - 'A'...'F' => { - self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 4; - }, - 'a'...'f' => { - self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 4; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - self.state = .string_backslash_u_3; - continue :state_loop; - }, - .string_backslash_u_3 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[0] |= c - '0'; - }, - 'A'...'F' => { - self.utf16_code_units[0] |= c - 'A' + 10; - }, - 'a'...'f' => { - self.utf16_code_units[0] |= c - 'a' + 10; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - if (std.unicode.utf16IsHighSurrogate(self.utf16_code_units[0])) { - self.state = .string_surrogate_half; - continue :state_loop; - } else if (std.unicode.utf16IsLowSurrogate(self.utf16_code_units[0])) { - return error.SyntaxError; // Unexpected low surrogate half. - } else { - self.value_start = self.cursor; - self.state = .string; - return partialStringCodepoint(self.utf16_code_units[0]); - } - }, - .string_surrogate_half => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - '\\' => { - self.cursor += 1; - self.state = .string_surrogate_half_backslash; - continue :state_loop; - }, - else => return error.SyntaxError, // Expected low surrogate half. - } - }, - .string_surrogate_half_backslash => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 'u' => { - self.cursor += 1; - self.state = .string_surrogate_half_backslash_u; - continue :state_loop; - }, - else => return error.SyntaxError, // Expected low surrogate half. - } - }, - .string_surrogate_half_backslash_u => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 'D', 'd' => { - self.cursor += 1; - self.utf16_code_units[1] = 0xD << 12; - self.state = .string_surrogate_half_backslash_u_1; - continue :state_loop; - }, - else => return error.SyntaxError, // Expected low surrogate half. - } - }, - .string_surrogate_half_backslash_u_1 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - 'C'...'F' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 8; - self.state = .string_surrogate_half_backslash_u_2; - continue :state_loop; - }, - 'c'...'f' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 8; - self.state = .string_surrogate_half_backslash_u_2; - continue :state_loop; - }, - else => return error.SyntaxError, // Expected low surrogate half. - } - }, - .string_surrogate_half_backslash_u_2 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - '0') << 4; - self.state = .string_surrogate_half_backslash_u_3; - continue :state_loop; - }, - 'A'...'F' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 4; - self.state = .string_surrogate_half_backslash_u_3; - continue :state_loop; - }, - 'a'...'f' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 4; - self.state = .string_surrogate_half_backslash_u_3; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .string_surrogate_half_backslash_u_3 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[1] |= c - '0'; - }, - 'A'...'F' => { - self.utf16_code_units[1] |= c - 'A' + 10; - }, - 'a'...'f' => { - self.utf16_code_units[1] |= c - 'a' + 10; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - const code_point = std.unicode.utf16DecodeSurrogatePair(&self.utf16_code_units) catch unreachable; - return partialStringCodepoint(code_point); - }, - - .string_utf8_last_byte => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0xBF => { - self.cursor += 1; - self.state = .string; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_second_to_last_byte => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0xBF => { - self.cursor += 1; - self.state = .string_utf8_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_second_to_last_byte_guard_against_overlong => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0xA0...0xBF => { - self.cursor += 1; - self.state = .string_utf8_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_second_to_last_byte_guard_against_surrogate_half => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0x9F => { - self.cursor += 1; - self.state = .string_utf8_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_third_to_last_byte => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0xBF => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_third_to_last_byte_guard_against_overlong => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x90...0xBF => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_third_to_last_byte_guard_against_too_large => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0x8F => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - - .literal_t => { - switch (try self.expectByte()) { - 'r' => { - self.cursor += 1; - self.state = .literal_tr; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_tr => { - switch (try self.expectByte()) { - 'u' => { - self.cursor += 1; - self.state = .literal_tru; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_tru => { - switch (try self.expectByte()) { - 'e' => { - self.cursor += 1; - self.state = .post_value; - return .true; - }, - else => return error.SyntaxError, - } - }, - .literal_f => { - switch (try self.expectByte()) { - 'a' => { - self.cursor += 1; - self.state = .literal_fa; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_fa => { - switch (try self.expectByte()) { - 'l' => { - self.cursor += 1; - self.state = .literal_fal; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_fal => { - switch (try self.expectByte()) { - 's' => { - self.cursor += 1; - self.state = .literal_fals; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_fals => { - switch (try self.expectByte()) { - 'e' => { - self.cursor += 1; - self.state = .post_value; - return .false; - }, - else => return error.SyntaxError, - } - }, - .literal_n => { - switch (try self.expectByte()) { - 'u' => { - self.cursor += 1; - self.state = .literal_nu; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_nu => { - switch (try self.expectByte()) { - 'l' => { - self.cursor += 1; - self.state = .literal_nul; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_nul => { - switch (try self.expectByte()) { - 'l' => { - self.cursor += 1; - self.state = .post_value; - return .null; - }, - else => return error.SyntaxError, - } - }, - } - unreachable; - } - } - - /// Seeks ahead in the input until the first byte of the next token (or the end of the input) - /// determines which type of token will be returned from the next `next*()` call. - /// This function is idempotent, only advancing past commas, colons, and inter-token whitespace. - pub fn peekNextTokenType(self: *@This()) PeekError!TokenType { - state_loop: while (true) { - switch (self.state) { - .value => { - switch (try self.skipWhitespaceExpectByte()) { - '{' => return .object_begin, - '[' => return .array_begin, - '"' => return .string, - '-', '0'...'9' => return .number, - 't' => return .true, - 'f' => return .false, - 'n' => return .null, - else => return error.SyntaxError, - } - }, - - .post_value => { - if (try self.skipWhitespaceCheckEnd()) return .end_of_document; - - const c = self.input[self.cursor]; - if (self.string_is_object_key) { - self.string_is_object_key = false; - switch (c) { - ':' => { - self.cursor += 1; - self.state = .value; - continue :state_loop; - }, - else => return error.SyntaxError, - } - } - - switch (c) { - '}' => return .object_end, - ']' => return .array_end, - ',' => { - switch (self.stack.peek()) { - OBJECT_MODE => { - self.state = .object_post_comma; - }, - ARRAY_MODE => { - self.state = .value; - }, - } - self.cursor += 1; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - - .object_start => { - switch (try self.skipWhitespaceExpectByte()) { - '"' => return .string, - '}' => return .object_end, - else => return error.SyntaxError, - } - }, - .object_post_comma => { - switch (try self.skipWhitespaceExpectByte()) { - '"' => return .string, - else => return error.SyntaxError, - } - }, - - .array_start => { - switch (try self.skipWhitespaceExpectByte()) { - ']' => return .array_end, - else => { - self.state = .value; - continue :state_loop; - }, - } - }, - - .number_minus, - .number_leading_zero, - .number_int, - .number_post_dot, - .number_frac, - .number_post_e, - .number_post_e_sign, - .number_exp, - => return .number, - - .string, - .string_backslash, - .string_backslash_u, - .string_backslash_u_1, - .string_backslash_u_2, - .string_backslash_u_3, - .string_surrogate_half, - .string_surrogate_half_backslash, - .string_surrogate_half_backslash_u, - .string_surrogate_half_backslash_u_1, - .string_surrogate_half_backslash_u_2, - .string_surrogate_half_backslash_u_3, - => return .string, - - .string_utf8_last_byte, - .string_utf8_second_to_last_byte, - .string_utf8_second_to_last_byte_guard_against_overlong, - .string_utf8_second_to_last_byte_guard_against_surrogate_half, - .string_utf8_third_to_last_byte, - .string_utf8_third_to_last_byte_guard_against_overlong, - .string_utf8_third_to_last_byte_guard_against_too_large, - => return .string, - - .literal_t, - .literal_tr, - .literal_tru, - => return .true, - .literal_f, - .literal_fa, - .literal_fal, - .literal_fals, - => return .false, - .literal_n, - .literal_nu, - .literal_nul, - => return .null, - } - unreachable; - } - } - - const State = enum { - value, - post_value, - - object_start, - object_post_comma, - - array_start, - - number_minus, - number_leading_zero, - number_int, - number_post_dot, - number_frac, - number_post_e, - number_post_e_sign, - number_exp, - - string, - string_backslash, - string_backslash_u, - string_backslash_u_1, - string_backslash_u_2, - string_backslash_u_3, - string_surrogate_half, - string_surrogate_half_backslash, - string_surrogate_half_backslash_u, - string_surrogate_half_backslash_u_1, - string_surrogate_half_backslash_u_2, - string_surrogate_half_backslash_u_3, - - // From http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String - string_utf8_last_byte, // State A - string_utf8_second_to_last_byte, // State B - string_utf8_second_to_last_byte_guard_against_overlong, // State C - string_utf8_second_to_last_byte_guard_against_surrogate_half, // State D - string_utf8_third_to_last_byte, // State E - string_utf8_third_to_last_byte_guard_against_overlong, // State F - string_utf8_third_to_last_byte_guard_against_too_large, // State G - - literal_t, - literal_tr, - literal_tru, - literal_f, - literal_fa, - literal_fal, - literal_fals, - literal_n, - literal_nu, - literal_nul, - }; - - fn expectByte(self: *const @This()) !u8 { - if (self.cursor < self.input.len) { - return self.input[self.cursor]; - } - // No byte. - if (self.is_end_of_input) return error.UnexpectedEndOfInput; - return error.BufferUnderrun; - } - - fn skipWhitespace(self: *@This()) void { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - // Whitespace - ' ', '\t', '\r' => continue, - '\n' => { - if (self.diagnostics) |diag| { - diag.line_number += 1; - // This will count the newline itself, - // which means a straight-forward subtraction will give a 1-based column number. - diag.line_start_cursor = self.cursor; - } - continue; - }, - else => return, - } - } - } - - fn skipWhitespaceExpectByte(self: *@This()) !u8 { - self.skipWhitespace(); - return self.expectByte(); - } - - fn skipWhitespaceCheckEnd(self: *@This()) !bool { - self.skipWhitespace(); - if (self.cursor >= self.input.len) { - // End of buffer. - if (self.is_end_of_input) { - // End of everything. - if (self.stackHeight() == 0) { - // We did it! - return true; - } - return error.UnexpectedEndOfInput; - } - return error.BufferUnderrun; - } - if (self.stackHeight() == 0) return error.SyntaxError; - return false; - } - - fn takeValueSlice(self: *@This()) []const u8 { - const slice = self.input[self.value_start..self.cursor]; - self.value_start = self.cursor; - return slice; - } - fn takeValueSliceMinusTrailingOffset(self: *@This(), trailing_negative_offset: usize) []const u8 { - // Check if the escape sequence started before the current input buffer. - // (The algebra here is awkward to avoid unsigned underflow, - // but it's just making sure the slice on the next line isn't UB.) - if (self.cursor <= self.value_start + trailing_negative_offset) return ""; - const slice = self.input[self.value_start .. self.cursor - trailing_negative_offset]; - // When trailing_negative_offset is non-zero, setting self.value_start doesn't matter, - // because we always set it again while emitting the .partial_string_escaped_*. - self.value_start = self.cursor; - return slice; - } - - fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token { - const slice = self.takeValueSlice(); - if (self.is_end_of_input) { - if (!allow_end) return error.UnexpectedEndOfInput; - self.state = .post_value; - return Token{ .number = slice }; - } - if (slice.len == 0) return error.BufferUnderrun; - return Token{ .partial_number = slice }; - } - - fn endOfBufferInString(self: *@This()) !Token { - if (self.is_end_of_input) return error.UnexpectedEndOfInput; - const slice = self.takeValueSliceMinusTrailingOffset(switch (self.state) { - // Don't include the escape sequence in the partial string. - .string_backslash => 1, - .string_backslash_u => 2, - .string_backslash_u_1 => 3, - .string_backslash_u_2 => 4, - .string_backslash_u_3 => 5, - .string_surrogate_half => 6, - .string_surrogate_half_backslash => 7, - .string_surrogate_half_backslash_u => 8, - .string_surrogate_half_backslash_u_1 => 9, - .string_surrogate_half_backslash_u_2 => 10, - .string_surrogate_half_backslash_u_3 => 11, - - // Include everything up to the cursor otherwise. - .string, - .string_utf8_last_byte, - .string_utf8_second_to_last_byte, - .string_utf8_second_to_last_byte_guard_against_overlong, - .string_utf8_second_to_last_byte_guard_against_surrogate_half, - .string_utf8_third_to_last_byte, - .string_utf8_third_to_last_byte_guard_against_overlong, - .string_utf8_third_to_last_byte_guard_against_too_large, - => 0, - - else => unreachable, - }); - if (slice.len == 0) return error.BufferUnderrun; - return Token{ .partial_string = slice }; - } - - fn partialStringCodepoint(code_point: u21) Token { - var buf: [4]u8 = undefined; - switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) { - 1 => return Token{ .partial_string_escaped_1 = buf[0..1].* }, - 2 => return Token{ .partial_string_escaped_2 = buf[0..2].* }, - 3 => return Token{ .partial_string_escaped_3 = buf[0..3].* }, - 4 => return Token{ .partial_string_escaped_4 = buf[0..4].* }, - else => unreachable, - } - } -}; - -const OBJECT_MODE = 0; -const ARRAY_MODE = 1; - -fn appendSlice(list: *std.ArrayList(u8), buf: []const u8, max_value_len: usize) !void { - const new_len = std.math.add(usize, list.items.len, buf.len) catch return error.ValueTooLong; - if (new_len > max_value_len) return error.ValueTooLong; - try list.appendSlice(buf); -} - -/// For the slice you get from a `Token.number` or `Token.allocated_number`, -/// this function returns true if the number doesn't contain any fraction or exponent components, and is not `-0`. -/// Note, the numeric value encoded by the value may still be an integer, such as `1.0`. -/// This function is meant to give a hint about whether integer parsing or float parsing should be used on the value. -/// This function will not give meaningful results on non-numeric input. -pub fn isNumberFormattedLikeAnInteger(value: []const u8) bool { - if (std.mem.eql(u8, value, "-0")) return false; - return std.mem.indexOfAny(u8, value, ".eE") == null; -} - -test { - _ = @import("./scanner_test.zig"); -} diff --git a/lib/std/json/scanner_test.zig b/lib/std/json/scanner_test.zig index d085cb661a7beafc8ee640f0180bf80c822b1273..eb5d5cb75ec29f6a501aa12e60c1f6b7fb65ca7b 100644 --- a/lib/std/json/scanner_test.zig +++ b/lib/std/json/scanner_test.zig @@ -1,13 +1,11 @@ const std = @import("std"); -const JsonScanner = @import("./scanner.zig").Scanner; -const jsonReader = @import("./scanner.zig").reader; -const JsonReader = @import("./scanner.zig").Reader; -const Token = @import("./scanner.zig").Token; -const TokenType = @import("./scanner.zig").TokenType; -const Diagnostics = @import("./scanner.zig").Diagnostics; -const Error = @import("./scanner.zig").Error; -const validate = @import("./scanner.zig").validate; -const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger; +const Scanner = @import("Scanner.zig"); +const Token = Scanner.Token; +const TokenType = Scanner.TokenType; +const Diagnostics = Scanner.Diagnostics; +const Error = Scanner.Error; +const validate = Scanner.validate; +const isNumberFormattedLikeAnInteger = Scanner.isNumberFormattedLikeAnInteger; const example_document_str = \\{ @@ -36,7 +34,7 @@ fn expectPeekNext(scanner_or_reader: anytype, expected_token_type: TokenType, ex } test "token" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, example_document_str); + var scanner = Scanner.initCompleteInput(std.testing.allocator, example_document_str); defer scanner.deinit(); try expectNext(&scanner, .object_begin); @@ -138,23 +136,25 @@ fn testAllTypes(source: anytype, large_buffer: bool) !void { } test "peek all types" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, all_types_test_case); + var scanner = Scanner.initCompleteInput(std.testing.allocator, all_types_test_case); defer scanner.deinit(); try testAllTypes(&scanner, true); - var stream = std.io.fixedBufferStream(all_types_test_case); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(all_types_test_case); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); try testAllTypes(&json_reader, true); - var tiny_stream = std.io.fixedBufferStream(all_types_test_case); - var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader()); + var tiny_buffer: [1]u8 = undefined; + var tiny_stream: std.testing.Reader = .init(&tiny_buffer, &.{.{ .buffer = all_types_test_case }}); + tiny_stream.artificial_limit = .limited(1); + var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream.interface); defer tiny_json_reader.deinit(); try testAllTypes(&tiny_json_reader, false); } test "token mismatched close" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, "[102, 111, 111 }"); + var scanner = Scanner.initCompleteInput(std.testing.allocator, "[102, 111, 111 }"); defer scanner.deinit(); try expectNext(&scanner, .array_begin); try expectNext(&scanner, Token{ .number = "102" }); @@ -164,15 +164,15 @@ test "token mismatched close" { } test "token premature object close" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, "{ \"key\": }"); + var scanner = Scanner.initCompleteInput(std.testing.allocator, "{ \"key\": }"); defer scanner.deinit(); try expectNext(&scanner, .object_begin); try expectNext(&scanner, Token{ .string = "key" }); try std.testing.expectError(error.SyntaxError, scanner.next()); } -test "JsonScanner basic" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, example_document_str); +test "Scanner basic" { + var scanner = Scanner.initCompleteInput(std.testing.allocator, example_document_str); defer scanner.deinit(); while (true) { @@ -181,10 +181,10 @@ test "JsonScanner basic" { } } -test "JsonReader basic" { - var stream = std.io.fixedBufferStream(example_document_str); +test "Scanner.Reader basic" { + var stream: std.Io.Reader = .fixed(example_document_str); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); while (true) { @@ -215,7 +215,7 @@ const number_test_items = blk: { test "numbers" { for (number_test_items) |number_str| { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, number_str); + var scanner = Scanner.initCompleteInput(std.testing.allocator, number_str); defer scanner.deinit(); const token = try scanner.next(); @@ -243,10 +243,10 @@ const string_test_cases = .{ test "strings" { inline for (string_test_cases) |tuple| { - var stream = std.io.fixedBufferStream("\"" ++ tuple[0] ++ "\""); + var stream: std.Io.Reader = .fixed("\"" ++ tuple[0] ++ "\""); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); const token = try json_reader.nextAlloc(arena.allocator(), .alloc_if_needed); @@ -289,7 +289,7 @@ test "nesting" { } fn expectMaybeError(document_str: []const u8, maybe_error: ?Error) !void { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, document_str); + var scanner = Scanner.initCompleteInput(std.testing.allocator, document_str); defer scanner.deinit(); while (true) { @@ -352,12 +352,12 @@ fn expectEqualTokens(expected_token: Token, actual_token: Token) !void { } fn testTinyBufferSize(document_str: []const u8) !void { - var tiny_stream = std.io.fixedBufferStream(document_str); - var normal_stream = std.io.fixedBufferStream(document_str); + var tiny_stream: std.Io.Reader = .fixed(document_str); + var normal_stream: std.Io.Reader = .fixed(document_str); - var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader()); + var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream); defer tiny_json_reader.deinit(); - var normal_json_reader = JsonReader(0x1000, @TypeOf(normal_stream.reader())).init(std.testing.allocator, normal_stream.reader()); + var normal_json_reader: Scanner.Reader = .init(std.testing.allocator, &normal_stream); defer normal_json_reader.deinit(); expectEqualStreamOfTokens(&normal_json_reader, &tiny_json_reader) catch |err| { @@ -397,13 +397,13 @@ test "validate" { } fn testSkipValue(s: []const u8) !void { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, s); + var scanner = Scanner.initCompleteInput(std.testing.allocator, s); defer scanner.deinit(); try scanner.skipValue(); try expectEqualTokens(.end_of_document, try scanner.next()); - var stream = std.io.fixedBufferStream(s); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(s); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); try json_reader.skipValue(); try expectEqualTokens(.end_of_document, try json_reader.next()); @@ -441,7 +441,7 @@ fn testEnsureStackCapacity(do_ensure: bool) !void { try input_string.appendNTimes(std.testing.allocator, ']', nestings); defer input_string.deinit(std.testing.allocator); - var scanner = JsonScanner.initCompleteInput(failing_allocator, input_string.items); + var scanner = Scanner.initCompleteInput(failing_allocator, input_string.items); defer scanner.deinit(); if (do_ensure) { @@ -473,17 +473,17 @@ fn testDiagnosticsFromSource(expected_error: ?anyerror, line: u64, col: u64, byt try std.testing.expectEqual(byte_offset, diagnostics.getByteOffset()); } fn testDiagnostics(expected_error: ?anyerror, line: u64, col: u64, byte_offset: u64, s: []const u8) !void { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, s); + var scanner = Scanner.initCompleteInput(std.testing.allocator, s); defer scanner.deinit(); try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &scanner); - var tiny_stream = std.io.fixedBufferStream(s); - var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader()); + var tiny_stream: std.Io.Reader = .fixed(s); + var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream); defer tiny_json_reader.deinit(); try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &tiny_json_reader); - var medium_stream = std.io.fixedBufferStream(s); - var medium_json_reader = JsonReader(5, @TypeOf(medium_stream.reader())).init(std.testing.allocator, medium_stream.reader()); + var medium_stream: std.Io.Reader = .fixed(s); + var medium_json_reader: Scanner.Reader = .init(std.testing.allocator, &medium_stream); defer medium_json_reader.deinit(); try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &medium_json_reader); } diff --git a/lib/std/json/static.zig b/lib/std/json/static.zig index 2504d591007f495e2296ca6fd879a888ad1909bf..44469adf4c60172c793b2a6fcd4a3ceabae1bfe7 100644 --- a/lib/std/json/static.zig +++ b/lib/std/json/static.zig @@ -4,11 +4,11 @@ const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; const ArrayList = std.ArrayList; -const Scanner = @import("./scanner.zig").Scanner; -const Token = @import("./scanner.zig").Token; -const AllocWhen = @import("./scanner.zig").AllocWhen; -const default_max_value_len = @import("./scanner.zig").default_max_value_len; -const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger; +const Scanner = @import("Scanner.zig"); +const Token = Scanner.Token; +const AllocWhen = Scanner.AllocWhen; +const default_max_value_len = Scanner.default_max_value_len; +const isNumberFormattedLikeAnInteger = Scanner.isNumberFormattedLikeAnInteger; const Value = @import("./dynamic.zig").Value; const Array = @import("./dynamic.zig").Array; diff --git a/lib/std/json/static_test.zig b/lib/std/json/static_test.zig index 3a1919e40c0016b2c980e422c14642b90699559d..735ccd82e16fef5f05c42d98673fb8e79d9de95f 100644 --- a/lib/std/json/static_test.zig +++ b/lib/std/json/static_test.zig @@ -12,9 +12,7 @@ const parseFromValue = @import("./static.zig").parseFromValue; const parseFromValueLeaky = @import("./static.zig").parseFromValueLeaky; const ParseOptions = @import("./static.zig").ParseOptions; -const JsonScanner = @import("./scanner.zig").Scanner; -const jsonReader = @import("./scanner.zig").reader; -const Diagnostics = @import("./scanner.zig").Diagnostics; +const Scanner = @import("Scanner.zig"); const Value = @import("./dynamic.zig").Value; @@ -300,9 +298,9 @@ const subnamespaces_0_doc = fn testAllParseFunctions(comptime T: type, expected: T, doc: []const u8) !void { // First do the one with the debug info in case we get a SyntaxError or something. { - var scanner = JsonScanner.initCompleteInput(testing.allocator, doc); + var scanner = Scanner.initCompleteInput(testing.allocator, doc); defer scanner.deinit(); - var diagnostics = Diagnostics{}; + var diagnostics = Scanner.Diagnostics{}; scanner.enableDiagnostics(&diagnostics); var parsed = parseFromTokenSource(T, testing.allocator, &scanner, .{}) catch |e| { std.debug.print("at line,col: {}:{}\n", .{ diagnostics.getLine(), diagnostics.getColumn() }); @@ -317,8 +315,8 @@ fn testAllParseFunctions(comptime T: type, expected: T, doc: []const u8) !void { try testing.expectEqualDeep(expected, parsed.value); } { - var stream = std.io.fixedBufferStream(doc); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(doc); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); var parsed = try parseFromTokenSource(T, testing.allocator, &json_reader, .{}); defer parsed.deinit(); @@ -331,13 +329,13 @@ fn testAllParseFunctions(comptime T: type, expected: T, doc: []const u8) !void { try testing.expectEqualDeep(expected, try parseFromSliceLeaky(T, arena.allocator(), doc, .{})); } { - var scanner = JsonScanner.initCompleteInput(testing.allocator, doc); + var scanner = Scanner.initCompleteInput(testing.allocator, doc); defer scanner.deinit(); try testing.expectEqualDeep(expected, try parseFromTokenSourceLeaky(T, arena.allocator(), &scanner, .{})); } { - var stream = std.io.fixedBufferStream(doc); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(doc); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); try testing.expectEqualDeep(expected, try parseFromTokenSourceLeaky(T, arena.allocator(), &json_reader, .{})); } @@ -763,7 +761,7 @@ test "parse exponential into int" { test "parseFromTokenSource" { { - var scanner = JsonScanner.initCompleteInput(testing.allocator, "123"); + var scanner = Scanner.initCompleteInput(testing.allocator, "123"); defer scanner.deinit(); var parsed = try parseFromTokenSource(u32, testing.allocator, &scanner, .{}); defer parsed.deinit(); @@ -771,8 +769,8 @@ test "parseFromTokenSource" { } { - var stream = std.io.fixedBufferStream("123"); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed("123"); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); var parsed = try parseFromTokenSource(u32, testing.allocator, &json_reader, .{}); defer parsed.deinit(); @@ -836,7 +834,7 @@ test "json parse partial" { \\} ; const allocator = testing.allocator; - var scanner = JsonScanner.initCompleteInput(allocator, str); + var scanner = Scanner.initCompleteInput(allocator, str); defer scanner.deinit(); var arena = ArenaAllocator.init(allocator); @@ -886,8 +884,8 @@ test "json parse allocate when streaming" { var arena = ArenaAllocator.init(allocator); defer arena.deinit(); - var stream = std.io.fixedBufferStream(str); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(str); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); const parsed = parseFromTokenSourceLeaky(T, arena.allocator(), &json_reader, .{}) catch |err| { json_reader.deinit(); diff --git a/lib/std/json/stringify.zig b/lib/std/json/stringify.zig deleted file mode 100644 index aa495736954849facb2e7ddd3e28306794950e59..0000000000000000000000000000000000000000 --- a/lib/std/json/stringify.zig +++ /dev/null @@ -1,772 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; -const ArrayList = std.ArrayList; -const BitStack = std.BitStack; - -const OBJECT_MODE = 0; -const ARRAY_MODE = 1; - -pub const StringifyOptions = struct { - /// Controls the whitespace emitted. - /// The default `.minified` is a compact encoding with no whitespace between tokens. - /// Any setting other than `.minified` will use newlines, indentation, and a space after each ':'. - /// `.indent_1` means 1 space for each indentation level, `.indent_2` means 2 spaces, etc. - /// `.indent_tab` uses a tab for each indentation level. - whitespace: enum { - minified, - indent_1, - indent_2, - indent_3, - indent_4, - indent_8, - indent_tab, - } = .minified, - - /// Should optional fields with null value be written? - emit_null_optional_fields: bool = true, - - /// Arrays/slices of u8 are typically encoded as JSON strings. - /// This option emits them as arrays of numbers instead. - /// Does not affect calls to `objectField*()`. - emit_strings_as_arrays: bool = false, - - /// Should unicode characters be escaped in strings? - escape_unicode: bool = false, - - /// When true, renders numbers outside the range `+-1<<53` (the precise integer range of f64) as JSON strings in base 10. - emit_nonportable_numbers_as_strings: bool = false, -}; - -/// Writes the given value to the `std.io.GenericWriter` stream. -/// See `WriteStream` for how the given value is serialized into JSON. -/// The maximum nesting depth of the output JSON document is 256. -/// See also `stringifyMaxDepth` and `stringifyArbitraryDepth`. -pub fn stringify( - value: anytype, - options: StringifyOptions, - out_stream: anytype, -) @TypeOf(out_stream).Error!void { - var jw = writeStream(out_stream, options); - defer jw.deinit(); - try jw.write(value); -} - -/// Like `stringify` with configurable nesting depth. -/// `max_depth` is rounded up to the nearest multiple of 8. -/// Give `null` for `max_depth` to disable some safety checks and allow arbitrary nesting depth. -/// See `writeStreamMaxDepth` for more info. -pub fn stringifyMaxDepth( - value: anytype, - options: StringifyOptions, - out_stream: anytype, - comptime max_depth: ?usize, -) @TypeOf(out_stream).Error!void { - var jw = writeStreamMaxDepth(out_stream, options, max_depth); - try jw.write(value); -} - -/// Like `stringify` but takes an allocator to facilitate safety checks while allowing arbitrary nesting depth. -/// These safety checks can be helpful when debugging custom `jsonStringify` implementations; -/// See `WriteStream`. -pub fn stringifyArbitraryDepth( - allocator: Allocator, - value: anytype, - options: StringifyOptions, - out_stream: anytype, -) WriteStream(@TypeOf(out_stream), .checked_to_arbitrary_depth).Error!void { - var jw = writeStreamArbitraryDepth(allocator, out_stream, options); - defer jw.deinit(); - try jw.write(value); -} - -/// Calls `stringifyArbitraryDepth` and stores the result in dynamically allocated memory -/// instead of taking a `std.io.GenericWriter`. -/// -/// Caller owns returned memory. -pub fn stringifyAlloc( - allocator: Allocator, - value: anytype, - options: StringifyOptions, -) error{OutOfMemory}![]u8 { - var list = std.ArrayList(u8).init(allocator); - errdefer list.deinit(); - try stringifyArbitraryDepth(allocator, value, options, list.writer()); - return list.toOwnedSlice(); -} - -/// See `WriteStream` for documentation. -/// Equivalent to calling `writeStreamMaxDepth` with a depth of `256`. -/// -/// The caller does *not* need to call `deinit()` on the returned object. -pub fn writeStream( - out_stream: anytype, - options: StringifyOptions, -) WriteStream(@TypeOf(out_stream), .{ .checked_to_fixed_depth = 256 }) { - return writeStreamMaxDepth(out_stream, options, 256); -} - -/// See `WriteStream` for documentation. -/// The returned object includes 1 bit of size per `max_depth` to enable safety checks on the order of method calls; -/// see the grammar in the `WriteStream` documentation. -/// `max_depth` is rounded up to the nearest multiple of 8. -/// If the nesting depth exceeds `max_depth`, it is detectable illegal behavior. -/// Give `null` for `max_depth` to disable safety checks for the grammar and allow arbitrary nesting depth. -/// In `ReleaseFast` and `ReleaseSmall`, `max_depth` is ignored, effectively equivalent to passing `null`. -/// Alternatively, see `writeStreamArbitraryDepth` to do safety checks to arbitrary depth. -/// -/// The caller does *not* need to call `deinit()` on the returned object. -pub fn writeStreamMaxDepth( - out_stream: anytype, - options: StringifyOptions, - comptime max_depth: ?usize, -) WriteStream( - @TypeOf(out_stream), - if (max_depth) |d| .{ .checked_to_fixed_depth = d } else .assumed_correct, -) { - return WriteStream( - @TypeOf(out_stream), - if (max_depth) |d| .{ .checked_to_fixed_depth = d } else .assumed_correct, - ).init(undefined, out_stream, options); -} - -/// See `WriteStream` for documentation. -/// This version of the write stream enables safety checks to arbitrarily deep nesting levels -/// by using the given allocator. -/// The caller should call `deinit()` on the returned object to free allocated memory. -/// -/// In `ReleaseFast` and `ReleaseSmall` mode, this function is effectively equivalent to calling `writeStreamMaxDepth(..., null)`; -/// in those build modes, the allocator is *not used*. -pub fn writeStreamArbitraryDepth( - allocator: Allocator, - out_stream: anytype, - options: StringifyOptions, -) WriteStream(@TypeOf(out_stream), .checked_to_arbitrary_depth) { - return WriteStream(@TypeOf(out_stream), .checked_to_arbitrary_depth).init(allocator, out_stream, options); -} - -/// Writes JSON ([RFC8259](https://tools.ietf.org/html/rfc8259)) formatted data -/// to a stream. -/// -/// The sequence of method calls to write JSON content must follow this grammar: -/// ``` -/// = -/// = -/// | -/// | -/// | write -/// | print -/// | -/// = beginObject ( )* endObject -/// = objectField | objectFieldRaw | -/// = beginArray ( )* endArray -/// = beginWriteRaw ( stream.writeAll )* endWriteRaw -/// = beginObjectFieldRaw ( stream.writeAll )* endObjectFieldRaw -/// ``` -/// -/// The `safety_checks_hint` parameter determines how much memory is used to enable assertions that the above grammar is being followed, -/// e.g. tripping an assertion rather than allowing `endObject` to emit the final `}` in `[[[]]}`. -/// "Depth" in this context means the depth of nested `[]` or `{}` expressions -/// (or equivalently the amount of recursion on the `` grammar expression above). -/// For example, emitting the JSON `[[[]]]` requires a depth of 3. -/// If `.checked_to_fixed_depth` is used, there is additionally an assertion that the nesting depth never exceeds the given limit. -/// `.checked_to_arbitrary_depth` requires a runtime allocator for the memory. -/// `.checked_to_fixed_depth` embeds the storage required in the `WriteStream` struct. -/// `.assumed_correct` requires no space and performs none of these assertions. -/// In `ReleaseFast` and `ReleaseSmall` mode, the given `safety_checks_hint` is ignored and is always treated as `.assumed_correct`. -pub fn WriteStream( - comptime OutStream: type, - comptime safety_checks_hint: union(enum) { - checked_to_arbitrary_depth, - checked_to_fixed_depth: usize, // Rounded up to the nearest multiple of 8. - assumed_correct, - }, -) type { - return struct { - const Self = @This(); - const build_mode_has_safety = switch (@import("builtin").mode) { - .Debug, .ReleaseSafe => true, - .ReleaseFast, .ReleaseSmall => false, - }; - const safety_checks: @TypeOf(safety_checks_hint) = if (build_mode_has_safety) - safety_checks_hint - else - .assumed_correct; - - pub const Stream = OutStream; - pub const Error = switch (safety_checks) { - .checked_to_arbitrary_depth => Stream.Error || error{OutOfMemory}, - .checked_to_fixed_depth, .assumed_correct => Stream.Error, - }; - - options: StringifyOptions, - - stream: OutStream, - indent_level: usize = 0, - next_punctuation: enum { - the_beginning, - none, - comma, - colon, - } = .the_beginning, - - nesting_stack: switch (safety_checks) { - .checked_to_arbitrary_depth => BitStack, - .checked_to_fixed_depth => |fixed_buffer_size| [(fixed_buffer_size + 7) >> 3]u8, - .assumed_correct => void, - }, - - raw_streaming_mode: if (build_mode_has_safety) - enum { none, value, objectField } - else - void = if (build_mode_has_safety) .none else {}, - - pub fn init(safety_allocator: Allocator, stream: OutStream, options: StringifyOptions) Self { - return .{ - .options = options, - .stream = stream, - .nesting_stack = switch (safety_checks) { - .checked_to_arbitrary_depth => BitStack.init(safety_allocator), - .checked_to_fixed_depth => |fixed_buffer_size| [_]u8{0} ** ((fixed_buffer_size + 7) >> 3), - .assumed_correct => {}, - }, - }; - } - - /// Only necessary with .checked_to_arbitrary_depth. - pub fn deinit(self: *Self) void { - switch (safety_checks) { - .checked_to_arbitrary_depth => self.nesting_stack.deinit(), - .checked_to_fixed_depth, .assumed_correct => {}, - } - self.* = undefined; - } - - pub fn beginArray(self: *Self) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - try self.valueStart(); - try self.stream.writeByte('['); - try self.pushIndentation(ARRAY_MODE); - self.next_punctuation = .none; - } - - pub fn beginObject(self: *Self) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - try self.valueStart(); - try self.stream.writeByte('{'); - try self.pushIndentation(OBJECT_MODE); - self.next_punctuation = .none; - } - - pub fn endArray(self: *Self) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - self.popIndentation(ARRAY_MODE); - switch (self.next_punctuation) { - .none => {}, - .comma => { - try self.indent(); - }, - .the_beginning, .colon => unreachable, - } - try self.stream.writeByte(']'); - self.valueDone(); - } - - pub fn endObject(self: *Self) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - self.popIndentation(OBJECT_MODE); - switch (self.next_punctuation) { - .none => {}, - .comma => { - try self.indent(); - }, - .the_beginning, .colon => unreachable, - } - try self.stream.writeByte('}'); - self.valueDone(); - } - - fn pushIndentation(self: *Self, mode: u1) !void { - switch (safety_checks) { - .checked_to_arbitrary_depth => { - try self.nesting_stack.push(mode); - self.indent_level += 1; - }, - .checked_to_fixed_depth => { - BitStack.pushWithStateAssumeCapacity(&self.nesting_stack, &self.indent_level, mode); - }, - .assumed_correct => { - self.indent_level += 1; - }, - } - } - fn popIndentation(self: *Self, assert_its_this_one: u1) void { - switch (safety_checks) { - .checked_to_arbitrary_depth => { - assert(self.nesting_stack.pop() == assert_its_this_one); - self.indent_level -= 1; - }, - .checked_to_fixed_depth => { - assert(BitStack.popWithState(&self.nesting_stack, &self.indent_level) == assert_its_this_one); - }, - .assumed_correct => { - self.indent_level -= 1; - }, - } - } - - fn indent(self: *Self) !void { - var char: u8 = ' '; - const n_chars = switch (self.options.whitespace) { - .minified => return, - .indent_1 => 1 * self.indent_level, - .indent_2 => 2 * self.indent_level, - .indent_3 => 3 * self.indent_level, - .indent_4 => 4 * self.indent_level, - .indent_8 => 8 * self.indent_level, - .indent_tab => blk: { - char = '\t'; - break :blk self.indent_level; - }, - }; - try self.stream.writeByte('\n'); - try self.stream.writeByteNTimes(char, n_chars); - } - - fn valueStart(self: *Self) !void { - if (self.isObjectKeyExpected()) |is_it| assert(!is_it); // Call objectField*(), not write(), for object keys. - return self.valueStartAssumeTypeOk(); - } - fn objectFieldStart(self: *Self) !void { - if (self.isObjectKeyExpected()) |is_it| assert(is_it); // Expected write(), not objectField*(). - return self.valueStartAssumeTypeOk(); - } - fn valueStartAssumeTypeOk(self: *Self) !void { - assert(!self.isComplete()); // JSON document already complete. - switch (self.next_punctuation) { - .the_beginning => { - // No indentation for the very beginning. - }, - .none => { - // First item in a container. - try self.indent(); - }, - .comma => { - // Subsequent item in a container. - try self.stream.writeByte(','); - try self.indent(); - }, - .colon => { - try self.stream.writeByte(':'); - if (self.options.whitespace != .minified) { - try self.stream.writeByte(' '); - } - }, - } - } - fn valueDone(self: *Self) void { - self.next_punctuation = .comma; - } - - // Only when safety is enabled: - fn isObjectKeyExpected(self: *const Self) ?bool { - switch (safety_checks) { - .checked_to_arbitrary_depth => return self.indent_level > 0 and - self.nesting_stack.peek() == OBJECT_MODE and - self.next_punctuation != .colon, - .checked_to_fixed_depth => return self.indent_level > 0 and - BitStack.peekWithState(&self.nesting_stack, self.indent_level) == OBJECT_MODE and - self.next_punctuation != .colon, - .assumed_correct => return null, - } - } - fn isComplete(self: *const Self) bool { - return self.indent_level == 0 and self.next_punctuation == .comma; - } - - /// An alternative to calling `write` that formats a value with `std.fmt`. - /// This function does the usual punctuation and indentation formatting - /// assuming the resulting formatted string represents a single complete value; - /// e.g. `"1"`, `"[]"`, `"[1,2]"`, not `"1,2"`. - /// This function may be useful for doing your own number formatting. - pub fn print(self: *Self, comptime fmt: []const u8, args: anytype) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - try self.valueStart(); - try self.stream.print(fmt, args); - self.valueDone(); - } - - /// An alternative to calling `write` that allows you to write directly to the `.stream` field, e.g. with `.stream.writeAll()`. - /// Call `beginWriteRaw()`, then write a complete value (including any quotes if necessary) directly to the `.stream` field, - /// then call `endWriteRaw()`. - /// This can be useful for streaming very long strings into the output without needing it all buffered in memory. - pub fn beginWriteRaw(self: *Self) !void { - if (build_mode_has_safety) { - assert(self.raw_streaming_mode == .none); - self.raw_streaming_mode = .value; - } - try self.valueStart(); - } - - /// See `beginWriteRaw`. - pub fn endWriteRaw(self: *Self) void { - if (build_mode_has_safety) { - assert(self.raw_streaming_mode == .value); - self.raw_streaming_mode = .none; - } - self.valueDone(); - } - - /// See `WriteStream` for when to call this method. - /// `key` is the string content of the property name. - /// Surrounding quotes will be added and any special characters will be escaped. - /// See also `objectFieldRaw`. - pub fn objectField(self: *Self, key: []const u8) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - try self.objectFieldStart(); - try encodeJsonString(key, self.options, self.stream); - self.next_punctuation = .colon; - } - /// See `WriteStream` for when to call this method. - /// `quoted_key` is the complete bytes of the key including quotes and any necessary escape sequences. - /// A few assertions are performed on the given value to ensure that the caller of this function understands the API contract. - /// See also `objectField`. - pub fn objectFieldRaw(self: *Self, quoted_key: []const u8) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - assert(quoted_key.len >= 2 and quoted_key[0] == '"' and quoted_key[quoted_key.len - 1] == '"'); // quoted_key should be "quoted". - try self.objectFieldStart(); - try self.stream.writeAll(quoted_key); - self.next_punctuation = .colon; - } - - /// In the rare case that you need to write very long object field names, - /// this is an alternative to `objectField` and `objectFieldRaw` that allows you to write directly to the `.stream` field - /// similar to `beginWriteRaw`. - /// Call `endObjectFieldRaw()` when you're done. - pub fn beginObjectFieldRaw(self: *Self) !void { - if (build_mode_has_safety) { - assert(self.raw_streaming_mode == .none); - self.raw_streaming_mode = .objectField; - } - try self.objectFieldStart(); - } - - /// See `beginObjectFieldRaw`. - pub fn endObjectFieldRaw(self: *Self) void { - if (build_mode_has_safety) { - assert(self.raw_streaming_mode == .objectField); - self.raw_streaming_mode = .none; - } - self.next_punctuation = .colon; - } - - /// Renders the given Zig value as JSON. - /// - /// Supported types: - /// * Zig `bool` -> JSON `true` or `false`. - /// * Zig `?T` -> `null` or the rendering of `T`. - /// * Zig `i32`, `u64`, etc. -> JSON number or string. - /// * When option `emit_nonportable_numbers_as_strings` is true, if the value is outside the range `+-1<<53` (the precise integer range of f64), it is rendered as a JSON string in base 10. Otherwise, it is rendered as JSON number. - /// * Zig floats -> JSON number or string. - /// * If the value cannot be precisely represented by an f64, it is rendered as a JSON string. Otherwise, it is rendered as JSON number. - /// * Zig `[]const u8`, `[]u8`, `*[N]u8`, `@Vector(N, u8)`, and similar -> JSON string. - /// * See `StringifyOptions.emit_strings_as_arrays`. - /// * If the content is not valid UTF-8, rendered as an array of numbers instead. - /// * Zig `[]T`, `[N]T`, `*[N]T`, `@Vector(N, T)`, and similar -> JSON array of the rendering of each item. - /// * Zig tuple -> JSON array of the rendering of each item. - /// * Zig `struct` -> JSON object with each field in declaration order. - /// * If the struct declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`. See `std.json.Value` for an example. - /// * See `StringifyOptions.emit_null_optional_fields`. - /// * Zig `union(enum)` -> JSON object with one field named for the active tag and a value representing the payload. - /// * If the payload is `void`, then the emitted value is `{}`. - /// * If the union declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`. - /// * Zig `enum` -> JSON string naming the active tag. - /// * If the enum declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`. - /// * If the enum is non-exhaustive, unnamed values are rendered as integers. - /// * Zig untyped enum literal -> JSON string naming the active tag. - /// * Zig error -> JSON string naming the error. - /// * Zig `*T` -> the rendering of `T`. Note there is no guard against circular-reference infinite recursion. - /// - /// See also alternative functions `print` and `beginWriteRaw`. - /// For writing object field names, use `objectField` instead. - pub fn write(self: *Self, value: anytype) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - const T = @TypeOf(value); - switch (@typeInfo(T)) { - .int => { - try self.valueStart(); - if (self.options.emit_nonportable_numbers_as_strings and - (value <= -(1 << 53) or value >= (1 << 53))) - { - try self.stream.print("\"{}\"", .{value}); - } else { - try self.stream.print("{}", .{value}); - } - self.valueDone(); - return; - }, - .comptime_int => { - return self.write(@as(std.math.IntFittingRange(value, value), value)); - }, - .float, .comptime_float => { - if (@as(f64, @floatCast(value)) == value) { - try self.valueStart(); - try self.stream.print("{}", .{@as(f64, @floatCast(value))}); - self.valueDone(); - return; - } - try self.valueStart(); - try self.stream.print("\"{}\"", .{value}); - self.valueDone(); - return; - }, - - .bool => { - try self.valueStart(); - try self.stream.writeAll(if (value) "true" else "false"); - self.valueDone(); - return; - }, - .null => { - try self.valueStart(); - try self.stream.writeAll("null"); - self.valueDone(); - return; - }, - .optional => { - if (value) |payload| { - return try self.write(payload); - } else { - return try self.write(null); - } - }, - .@"enum" => |enum_info| { - if (std.meta.hasFn(T, "jsonStringify")) { - return value.jsonStringify(self); - } - - if (!enum_info.is_exhaustive) { - inline for (enum_info.fields) |field| { - if (value == @field(T, field.name)) { - break; - } - } else { - return self.write(@intFromEnum(value)); - } - } - - return self.stringValue(@tagName(value)); - }, - .enum_literal => { - return self.stringValue(@tagName(value)); - }, - .@"union" => { - if (std.meta.hasFn(T, "jsonStringify")) { - return value.jsonStringify(self); - } - - const info = @typeInfo(T).@"union"; - if (info.tag_type) |UnionTagType| { - try self.beginObject(); - inline for (info.fields) |u_field| { - if (value == @field(UnionTagType, u_field.name)) { - try self.objectField(u_field.name); - if (u_field.type == void) { - // void value is {} - try self.beginObject(); - try self.endObject(); - } else { - try self.write(@field(value, u_field.name)); - } - break; - } - } else { - unreachable; // No active tag? - } - try self.endObject(); - return; - } else { - @compileError("Unable to stringify untagged union '" ++ @typeName(T) ++ "'"); - } - }, - .@"struct" => |S| { - if (std.meta.hasFn(T, "jsonStringify")) { - return value.jsonStringify(self); - } - - if (S.is_tuple) { - try self.beginArray(); - } else { - try self.beginObject(); - } - inline for (S.fields) |Field| { - // don't include void fields - if (Field.type == void) continue; - - var emit_field = true; - - // don't include optional fields that are null when emit_null_optional_fields is set to false - if (@typeInfo(Field.type) == .optional) { - if (self.options.emit_null_optional_fields == false) { - if (@field(value, Field.name) == null) { - emit_field = false; - } - } - } - - if (emit_field) { - if (!S.is_tuple) { - try self.objectField(Field.name); - } - try self.write(@field(value, Field.name)); - } - } - if (S.is_tuple) { - try self.endArray(); - } else { - try self.endObject(); - } - return; - }, - .error_set => return self.stringValue(@errorName(value)), - .pointer => |ptr_info| switch (ptr_info.size) { - .one => switch (@typeInfo(ptr_info.child)) { - .array => { - // Coerce `*[N]T` to `[]const T`. - const Slice = []const std.meta.Elem(ptr_info.child); - return self.write(@as(Slice, value)); - }, - else => { - return self.write(value.*); - }, - }, - .many, .slice => { - if (ptr_info.size == .many and ptr_info.sentinel() == null) - @compileError("unable to stringify type '" ++ @typeName(T) ++ "' without sentinel"); - const slice = if (ptr_info.size == .many) std.mem.span(value) else value; - - if (ptr_info.child == u8) { - // This is a []const u8, or some similar Zig string. - if (!self.options.emit_strings_as_arrays and std.unicode.utf8ValidateSlice(slice)) { - return self.stringValue(slice); - } - } - - try self.beginArray(); - for (slice) |x| { - try self.write(x); - } - try self.endArray(); - return; - }, - else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"), - }, - .array => { - // Coerce `[N]T` to `*const [N]T` (and then to `[]const T`). - return self.write(&value); - }, - .vector => |info| { - const array: [info.len]info.child = value; - return self.write(&array); - }, - else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"), - } - unreachable; - } - - fn stringValue(self: *Self, s: []const u8) !void { - try self.valueStart(); - try encodeJsonString(s, self.options, self.stream); - self.valueDone(); - } - }; -} - -fn outputUnicodeEscape(codepoint: u21, out_stream: anytype) !void { - if (codepoint <= 0xFFFF) { - // If the character is in the Basic Multilingual Plane (U+0000 through U+FFFF), - // then it may be represented as a six-character sequence: a reverse solidus, followed - // by the lowercase letter u, followed by four hexadecimal digits that encode the character's code point. - try out_stream.writeAll("\\u"); - //try w.printInt("x", .{ .width = 4, .fill = '0' }, codepoint); - try std.fmt.format(out_stream, "{x:0>4}", .{codepoint}); - } else { - assert(codepoint <= 0x10FFFF); - // To escape an extended character that is not in the Basic Multilingual Plane, - // the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair. - const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800; - const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00; - try out_stream.writeAll("\\u"); - //try w.printInt("x", .{ .width = 4, .fill = '0' }, high); - try std.fmt.format(out_stream, "{x:0>4}", .{high}); - try out_stream.writeAll("\\u"); - //try w.printInt("x", .{ .width = 4, .fill = '0' }, low); - try std.fmt.format(out_stream, "{x:0>4}", .{low}); - } -} - -fn outputSpecialEscape(c: u8, writer: anytype) !void { - switch (c) { - '\\' => try writer.writeAll("\\\\"), - '\"' => try writer.writeAll("\\\""), - 0x08 => try writer.writeAll("\\b"), - 0x0C => try writer.writeAll("\\f"), - '\n' => try writer.writeAll("\\n"), - '\r' => try writer.writeAll("\\r"), - '\t' => try writer.writeAll("\\t"), - else => try outputUnicodeEscape(c, writer), - } -} - -/// Write `string` to `writer` as a JSON encoded string. -pub fn encodeJsonString(string: []const u8, options: StringifyOptions, writer: anytype) !void { - try writer.writeByte('\"'); - try encodeJsonStringChars(string, options, writer); - try writer.writeByte('\"'); -} - -/// Write `chars` to `writer` as JSON encoded string characters. -pub fn encodeJsonStringChars(chars: []const u8, options: StringifyOptions, writer: anytype) !void { - var write_cursor: usize = 0; - var i: usize = 0; - if (options.escape_unicode) { - while (i < chars.len) : (i += 1) { - switch (chars[i]) { - // normal ascii character - 0x20...0x21, 0x23...0x5B, 0x5D...0x7E => {}, - 0x00...0x1F, '\\', '\"' => { - // Always must escape these. - try writer.writeAll(chars[write_cursor..i]); - try outputSpecialEscape(chars[i], writer); - write_cursor = i + 1; - }, - 0x7F...0xFF => { - try writer.writeAll(chars[write_cursor..i]); - const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable; - const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable; - try outputUnicodeEscape(codepoint, writer); - i += ulen - 1; - write_cursor = i + 1; - }, - } - } - } else { - while (i < chars.len) : (i += 1) { - switch (chars[i]) { - // normal bytes - 0x20...0x21, 0x23...0x5B, 0x5D...0xFF => {}, - 0x00...0x1F, '\\', '\"' => { - // Always must escape these. - try writer.writeAll(chars[write_cursor..i]); - try outputSpecialEscape(chars[i], writer); - write_cursor = i + 1; - }, - } - } - } - try writer.writeAll(chars[write_cursor..chars.len]); -} - -test { - _ = @import("./stringify_test.zig"); -} diff --git a/lib/std/json/stringify_test.zig b/lib/std/json/stringify_test.zig deleted file mode 100644 index 22dd50428593c453325e2dfeef709d8daf64bf17..0000000000000000000000000000000000000000 --- a/lib/std/json/stringify_test.zig +++ /dev/null @@ -1,504 +0,0 @@ -const std = @import("std"); -const mem = std.mem; -const testing = std.testing; - -const ObjectMap = @import("dynamic.zig").ObjectMap; -const Value = @import("dynamic.zig").Value; - -const StringifyOptions = @import("stringify.zig").StringifyOptions; -const stringify = @import("stringify.zig").stringify; -const stringifyMaxDepth = @import("stringify.zig").stringifyMaxDepth; -const stringifyArbitraryDepth = @import("stringify.zig").stringifyArbitraryDepth; -const stringifyAlloc = @import("stringify.zig").stringifyAlloc; -const writeStream = @import("stringify.zig").writeStream; -const writeStreamMaxDepth = @import("stringify.zig").writeStreamMaxDepth; -const writeStreamArbitraryDepth = @import("stringify.zig").writeStreamArbitraryDepth; - -test "json write stream" { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - { - var w = writeStream(out, .{ .whitespace = .indent_2 }); - try testBasicWriteStream(&w, &slice_stream); - } - - { - var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, 8); - try testBasicWriteStream(&w, &slice_stream); - } - - { - var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, null); - try testBasicWriteStream(&w, &slice_stream); - } - - { - var w = writeStreamArbitraryDepth(testing.allocator, out, .{ .whitespace = .indent_2 }); - defer w.deinit(); - try testBasicWriteStream(&w, &slice_stream); - } -} - -fn testBasicWriteStream(w: anytype, slice_stream: anytype) !void { - slice_stream.reset(); - - try w.beginObject(); - - try w.objectField("object"); - var arena_allocator = std.heap.ArenaAllocator.init(testing.allocator); - defer arena_allocator.deinit(); - try w.write(try getJsonObject(arena_allocator.allocator())); - - try w.objectFieldRaw("\"string\""); - try w.write("This is a string"); - - try w.objectField("array"); - try w.beginArray(); - try w.write("Another string"); - try w.write(@as(i32, 1)); - try w.write(@as(f32, 3.5)); - try w.endArray(); - - try w.objectField("int"); - try w.write(@as(i32, 10)); - - try w.objectField("float"); - try w.write(@as(f32, 3.5)); - - try w.endObject(); - - const result = slice_stream.getWritten(); - const expected = - \\{ - \\ "object": { - \\ "one": 1, - \\ "two": 2 - \\ }, - \\ "string": "This is a string", - \\ "array": [ - \\ "Another string", - \\ 1, - \\ 3.5 - \\ ], - \\ "int": 10, - \\ "float": 3.5 - \\} - ; - try std.testing.expectEqualStrings(expected, result); -} - -fn getJsonObject(allocator: std.mem.Allocator) !Value { - var value = Value{ .object = ObjectMap.init(allocator) }; - try value.object.put("one", Value{ .integer = @as(i64, @intCast(1)) }); - try value.object.put("two", Value{ .float = 2.0 }); - return value; -} - -test "stringify null optional fields" { - const MyStruct = struct { - optional: ?[]const u8 = null, - required: []const u8 = "something", - another_optional: ?[]const u8 = null, - another_required: []const u8 = "something else", - }; - try testStringify( - \\{"optional":null,"required":"something","another_optional":null,"another_required":"something else"} - , - MyStruct{}, - .{}, - ); - try testStringify( - \\{"required":"something","another_required":"something else"} - , - MyStruct{}, - .{ .emit_null_optional_fields = false }, - ); -} - -test "stringify basic types" { - try testStringify("false", false, .{}); - try testStringify("true", true, .{}); - try testStringify("null", @as(?u8, null), .{}); - try testStringify("null", @as(?*u32, null), .{}); - try testStringify("42", 42, .{}); - try testStringify("42", 42.0, .{}); - try testStringify("42", @as(u8, 42), .{}); - try testStringify("42", @as(u128, 42), .{}); - try testStringify("9999999999999999", 9999999999999999, .{}); - try testStringify("42", @as(f32, 42), .{}); - try testStringify("42", @as(f64, 42), .{}); - try testStringify("\"ItBroke\"", @as(anyerror, error.ItBroke), .{}); - try testStringify("\"ItBroke\"", error.ItBroke, .{}); -} - -test "stringify string" { - try testStringify("\"hello\"", "hello", .{}); - try testStringify("\"with\\nescapes\\r\"", "with\nescapes\r", .{}); - try testStringify("\"with\\nescapes\\r\"", "with\nescapes\r", .{ .escape_unicode = true }); - try testStringify("\"with unicode\\u0001\"", "with unicode\u{1}", .{}); - try testStringify("\"with unicode\\u0001\"", "with unicode\u{1}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{80}\"", "with unicode\u{80}", .{}); - try testStringify("\"with unicode\\u0080\"", "with unicode\u{80}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{FF}\"", "with unicode\u{FF}", .{}); - try testStringify("\"with unicode\\u00ff\"", "with unicode\u{FF}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{100}\"", "with unicode\u{100}", .{}); - try testStringify("\"with unicode\\u0100\"", "with unicode\u{100}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{800}\"", "with unicode\u{800}", .{}); - try testStringify("\"with unicode\\u0800\"", "with unicode\u{800}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{8000}\"", "with unicode\u{8000}", .{}); - try testStringify("\"with unicode\\u8000\"", "with unicode\u{8000}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{D799}\"", "with unicode\u{D799}", .{}); - try testStringify("\"with unicode\\ud799\"", "with unicode\u{D799}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{10000}\"", "with unicode\u{10000}", .{}); - try testStringify("\"with unicode\\ud800\\udc00\"", "with unicode\u{10000}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{10FFFF}\"", "with unicode\u{10FFFF}", .{}); - try testStringify("\"with unicode\\udbff\\udfff\"", "with unicode\u{10FFFF}", .{ .escape_unicode = true }); -} - -test "stringify many-item sentinel-terminated string" { - try testStringify("\"hello\"", @as([*:0]const u8, "hello"), .{}); - try testStringify("\"with\\nescapes\\r\"", @as([*:0]const u8, "with\nescapes\r"), .{ .escape_unicode = true }); - try testStringify("\"with unicode\\u0001\"", @as([*:0]const u8, "with unicode\u{1}"), .{ .escape_unicode = true }); -} - -test "stringify enums" { - const E = enum { - foo, - bar, - }; - try testStringify("\"foo\"", E.foo, .{}); - try testStringify("\"bar\"", E.bar, .{}); -} - -test "stringify non-exhaustive enum" { - const E = enum(u8) { - foo = 0, - _, - }; - try testStringify("\"foo\"", E.foo, .{}); - try testStringify("1", @as(E, @enumFromInt(1)), .{}); -} - -test "stringify enum literals" { - try testStringify("\"foo\"", .foo, .{}); - try testStringify("\"bar\"", .bar, .{}); -} - -test "stringify tagged unions" { - const T = union(enum) { - nothing, - foo: u32, - bar: bool, - }; - try testStringify("{\"nothing\":{}}", T{ .nothing = {} }, .{}); - try testStringify("{\"foo\":42}", T{ .foo = 42 }, .{}); - try testStringify("{\"bar\":true}", T{ .bar = true }, .{}); -} - -test "stringify struct" { - try testStringify("{\"foo\":42}", struct { - foo: u32, - }{ .foo = 42 }, .{}); -} - -test "emit_strings_as_arrays" { - // Should only affect string values, not object keys. - try testStringify("{\"foo\":\"bar\"}", .{ .foo = "bar" }, .{}); - try testStringify("{\"foo\":[98,97,114]}", .{ .foo = "bar" }, .{ .emit_strings_as_arrays = true }); - // Should *not* affect these types: - try testStringify("\"foo\"", @as(enum { foo, bar }, .foo), .{ .emit_strings_as_arrays = true }); - try testStringify("\"ItBroke\"", error.ItBroke, .{ .emit_strings_as_arrays = true }); - // Should work on these: - try testStringify("\"bar\"", @Vector(3, u8){ 'b', 'a', 'r' }, .{}); - try testStringify("[98,97,114]", @Vector(3, u8){ 'b', 'a', 'r' }, .{ .emit_strings_as_arrays = true }); - try testStringify("\"bar\"", [3]u8{ 'b', 'a', 'r' }, .{}); - try testStringify("[98,97,114]", [3]u8{ 'b', 'a', 'r' }, .{ .emit_strings_as_arrays = true }); -} - -test "stringify struct with indentation" { - try testStringify( - \\{ - \\ "foo": 42, - \\ "bar": [ - \\ 1, - \\ 2, - \\ 3 - \\ ] - \\} - , - struct { - foo: u32, - bar: [3]u32, - }{ - .foo = 42, - .bar = .{ 1, 2, 3 }, - }, - .{ .whitespace = .indent_4 }, - ); - try testStringify( - "{\n\t\"foo\": 42,\n\t\"bar\": [\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}", - struct { - foo: u32, - bar: [3]u32, - }{ - .foo = 42, - .bar = .{ 1, 2, 3 }, - }, - .{ .whitespace = .indent_tab }, - ); - try testStringify( - \\{"foo":42,"bar":[1,2,3]} - , - struct { - foo: u32, - bar: [3]u32, - }{ - .foo = 42, - .bar = .{ 1, 2, 3 }, - }, - .{ .whitespace = .minified }, - ); -} - -test "stringify struct with void field" { - try testStringify("{\"foo\":42}", struct { - foo: u32, - bar: void = {}, - }{ .foo = 42 }, .{}); -} - -test "stringify array of structs" { - const MyStruct = struct { - foo: u32, - }; - try testStringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ - MyStruct{ .foo = 42 }, - MyStruct{ .foo = 100 }, - MyStruct{ .foo = 1000 }, - }, .{}); -} - -test "stringify struct with custom stringifier" { - try testStringify("[\"something special\",42]", struct { - foo: u32, - const Self = @This(); - pub fn jsonStringify(value: @This(), jws: anytype) !void { - _ = value; - try jws.beginArray(); - try jws.write("something special"); - try jws.write(42); - try jws.endArray(); - } - }{ .foo = 42 }, .{}); -} - -test "stringify vector" { - try testStringify("[1,1]", @as(@Vector(2, u32), @splat(1)), .{}); - try testStringify("\"AA\"", @as(@Vector(2, u8), @splat('A')), .{}); - try testStringify("[65,65]", @as(@Vector(2, u8), @splat('A')), .{ .emit_strings_as_arrays = true }); -} - -test "stringify tuple" { - try testStringify("[\"foo\",42]", std.meta.Tuple(&.{ []const u8, usize }){ "foo", 42 }, .{}); -} - -fn testStringify(expected: []const u8, value: anytype, options: StringifyOptions) !void { - const ValidationWriter = struct { - const Self = @This(); - pub const Writer = std.io.GenericWriter(*Self, Error, write); - pub const Error = error{ - TooMuchData, - DifferentData, - }; - - expected_remaining: []const u8, - - fn init(exp: []const u8) Self { - return .{ .expected_remaining = exp }; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } - - fn write(self: *Self, bytes: []const u8) Error!usize { - if (self.expected_remaining.len < bytes.len) { - std.debug.print( - \\====== expected this output: ========= - \\{s} - \\======== instead found this: ========= - \\{s} - \\====================================== - , .{ - self.expected_remaining, - bytes, - }); - return error.TooMuchData; - } - if (!mem.eql(u8, self.expected_remaining[0..bytes.len], bytes)) { - std.debug.print( - \\====== expected this output: ========= - \\{s} - \\======== instead found this: ========= - \\{s} - \\====================================== - , .{ - self.expected_remaining[0..bytes.len], - bytes, - }); - return error.DifferentData; - } - self.expected_remaining = self.expected_remaining[bytes.len..]; - return bytes.len; - } - }; - - var vos = ValidationWriter.init(expected); - try stringifyArbitraryDepth(testing.allocator, value, options, vos.writer()); - if (vos.expected_remaining.len > 0) return error.NotEnoughData; - - // Also test with safety disabled. - try testStringifyMaxDepth(expected, value, options, null); - try testStringifyArbitraryDepth(expected, value, options); -} - -fn testStringifyMaxDepth(expected: []const u8, value: anytype, options: StringifyOptions, comptime max_depth: ?usize) !void { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - try stringifyMaxDepth(value, options, out, max_depth); - const got = slice_stream.getWritten(); - - try testing.expectEqualStrings(expected, got); -} - -fn testStringifyArbitraryDepth(expected: []const u8, value: anytype, options: StringifyOptions) !void { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - try stringifyArbitraryDepth(testing.allocator, value, options, out); - const got = slice_stream.getWritten(); - - try testing.expectEqualStrings(expected, got); -} - -test "stringify alloc" { - const allocator = std.testing.allocator; - const expected = - \\{"foo":"bar","answer":42,"my_friend":"sammy"} - ; - const actual = try stringifyAlloc(allocator, .{ .foo = "bar", .answer = 42, .my_friend = "sammy" }, .{}); - defer allocator.free(actual); - - try std.testing.expectEqualStrings(expected, actual); -} - -test "comptime stringify" { - comptime testStringifyMaxDepth("false", false, .{}, null) catch unreachable; - comptime testStringifyMaxDepth("false", false, .{}, 0) catch unreachable; - comptime testStringifyArbitraryDepth("false", false, .{}) catch unreachable; - - const MyStruct = struct { - foo: u32, - }; - comptime testStringifyMaxDepth("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ - MyStruct{ .foo = 42 }, - MyStruct{ .foo = 100 }, - MyStruct{ .foo = 1000 }, - }, .{}, null) catch unreachable; - comptime testStringifyMaxDepth("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ - MyStruct{ .foo = 42 }, - MyStruct{ .foo = 100 }, - MyStruct{ .foo = 1000 }, - }, .{}, 8) catch unreachable; -} - -test "print" { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - var w = writeStream(out, .{ .whitespace = .indent_2 }); - defer w.deinit(); - - try w.beginObject(); - try w.objectField("a"); - try w.print("[ ]", .{}); - try w.objectField("b"); - try w.beginArray(); - try w.print("[{s}] ", .{"[]"}); - try w.print(" {}", .{12345}); - try w.endArray(); - try w.endObject(); - - const result = slice_stream.getWritten(); - const expected = - \\{ - \\ "a": [ ], - \\ "b": [ - \\ [[]] , - \\ 12345 - \\ ] - \\} - ; - try std.testing.expectEqualStrings(expected, result); -} - -test "nonportable numbers" { - try testStringify("9999999999999999", 9999999999999999, .{}); - try testStringify("\"9999999999999999\"", 9999999999999999, .{ .emit_nonportable_numbers_as_strings = true }); -} - -test "stringify raw streaming" { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - { - var w = writeStream(out, .{ .whitespace = .indent_2 }); - try testRawStreaming(&w, &slice_stream); - } - - { - var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, 8); - try testRawStreaming(&w, &slice_stream); - } - - { - var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, null); - try testRawStreaming(&w, &slice_stream); - } - - { - var w = writeStreamArbitraryDepth(testing.allocator, out, .{ .whitespace = .indent_2 }); - defer w.deinit(); - try testRawStreaming(&w, &slice_stream); - } -} - -fn testRawStreaming(w: anytype, slice_stream: anytype) !void { - slice_stream.reset(); - - try w.beginObject(); - try w.beginObjectFieldRaw(); - try w.stream.writeAll("\"long"); - try w.stream.writeAll(" key\""); - w.endObjectFieldRaw(); - try w.beginWriteRaw(); - try w.stream.writeAll("\"long"); - try w.stream.writeAll(" value\""); - w.endWriteRaw(); - try w.endObject(); - - const result = slice_stream.getWritten(); - const expected = - \\{ - \\ "long key": "long value" - \\} - ; - try std.testing.expectEqualStrings(expected, result); -} diff --git a/lib/std/json/test.zig b/lib/std/json/test.zig index 136e8e34d18b25aebf3b4b0f8bce680a526b2b43..d3d803e93907babeb81d8106f2f725fa516b49a2 100644 --- a/lib/std/json/test.zig +++ b/lib/std/json/test.zig @@ -1,10 +1,9 @@ const std = @import("std"); +const json = std.json; const testing = std.testing; const parseFromSlice = @import("./static.zig").parseFromSlice; -const validate = @import("./scanner.zig").validate; -const JsonScanner = @import("./scanner.zig").Scanner; +const Scanner = @import("./Scanner.zig"); const Value = @import("./dynamic.zig").Value; -const stringifyAlloc = @import("./stringify.zig").stringifyAlloc; // Support for JSONTestSuite.zig pub fn ok(s: []const u8) !void { @@ -20,7 +19,7 @@ pub fn any(s: []const u8) !void { testHighLevelDynamicParser(s) catch {}; } fn testLowLevelScanner(s: []const u8) !void { - var scanner = JsonScanner.initCompleteInput(testing.allocator, s); + var scanner = Scanner.initCompleteInput(testing.allocator, s); defer scanner.deinit(); while (true) { const token = try scanner.next(); @@ -47,12 +46,12 @@ test "n_object_closed_missing_value" { } fn roundTrip(s: []const u8) !void { - try testing.expect(try validate(testing.allocator, s)); + try testing.expect(try Scanner.validate(testing.allocator, s)); var parsed = try parseFromSlice(Value, testing.allocator, s, .{}); defer parsed.deinit(); - const rendered = try stringifyAlloc(testing.allocator, parsed.value, .{}); + const rendered = try json.Stringify.valueAlloc(testing.allocator, parsed.value, .{}); defer testing.allocator.free(rendered); try testing.expectEqualStrings(s, rendered); -- 2.54.0 From b956b021878cd7b67f01c30a30fcb085887f24dc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 16:24:37 -0700 Subject: [PATCH 06/26] `zig env`: update std.json API --- src/print_env.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/print_env.zig b/src/print_env.zig index af433ddfc1eb9338c6bfa78e8c36d0474037502c..d1546abc6c64196031d9c6dcb43b0c6fc836fafa 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -21,10 +21,10 @@ pub fn cmdEnv(arena: Allocator, args: []const []const u8) !void { const host = try std.zig.system.resolveTargetQuery(.{}); const triple = try host.zigTriple(arena); - var bw = std.io.bufferedWriter(std.fs.File.stdout().deprecatedWriter()); - const w = bw.writer(); + var buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&buffer); - var jws = std.json.writeStream(w, .{ .whitespace = .indent_1 }); + var jws: std.json.Stringify = .{ .writer = &stdout_writer.interface, .options = .{ .whitespace = .indent_1 } }; try jws.beginObject(); @@ -55,7 +55,7 @@ pub fn cmdEnv(arena: Allocator, args: []const []const u8) !void { try jws.endObject(); try jws.endObject(); - try w.writeByte('\n'); - try bw.flush(); + try stdout_writer.interface.writeByte('\n'); + try stdout_writer.interface.flush(); } -- 2.54.0 From c3da98cf5a8b8a3164db78998bda848100871918 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 16:59:16 -0700 Subject: [PATCH 07/26] std.zon: update to new I/O API --- lib/std/Build/Cache/Path.zig | 8 +- lib/std/zig.zig | 29 +- lib/std/zig/Ast.zig | 2 +- lib/std/zon/parse.zig | 26 +- lib/std/zon/stringify.zig | 1869 +++++++++++++++++----------------- 5 files changed, 960 insertions(+), 974 deletions(-) diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index a0a58067fc00262966304c17c5d245ef1820e3d2..efd0f86105bcaecc0d1434424de4a2b56b1ecf0a 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -161,17 +161,19 @@ pub fn formatEscapeString(path: Path, writer: *std.io.Writer) std.io.Writer.Erro } } +/// Deprecated, use double quoted escape to print paths. pub fn fmtEscapeChar(path: Path) std.fmt.Formatter(Path, formatEscapeChar) { return .{ .data = path }; } +/// Deprecated, use double quoted escape to print paths. pub fn formatEscapeChar(path: Path, writer: *std.io.Writer) std.io.Writer.Error!void { if (path.root_dir.path) |p| { - try std.zig.charEscape(p, writer); - if (path.sub_path.len > 0) try std.zig.charEscape(fs.path.sep_str, writer); + for (p) |byte| try std.zig.charEscape(byte, writer); + if (path.sub_path.len > 0) try writer.writeByte(fs.path.sep); } if (path.sub_path.len > 0) { - try std.zig.charEscape(path.sub_path, writer); + for (path.sub_path) |byte| try std.zig.charEscape(byte, writer); } } diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 51eac1b0a6fa09855730c2d074421f0afda22198..4d75bfa2db6e8aea8d0f4b86fbe42862cb4cfadb 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -446,8 +446,8 @@ pub fn fmtString(bytes: []const u8) std.fmt.Formatter([]const u8, stringEscape) } /// Return a formatter for escaping a single quoted Zig string. -pub fn fmtChar(bytes: []const u8) std.fmt.Formatter([]const u8, charEscape) { - return .{ .data = bytes }; +pub fn fmtChar(c: u21) std.fmt.Formatter(u21, charEscape) { + return .{ .data = c }; } test fmtString { @@ -458,9 +458,7 @@ test fmtString { } test fmtChar { - try std.testing.expectFmt( - \\" \\ hi \x07 \x11 " derp \'" - , "\"{f}\"", .{fmtChar(" \\ hi \x07 \x11 \" derp '")}); + try std.testing.expectFmt("c \\u{26a1}", "{f} {f}", .{ fmtChar('c'), fmtChar('âš¡') }); } /// Print the string as escaped contents of a double quoted string. @@ -480,21 +478,26 @@ pub fn stringEscape(bytes: []const u8, w: *Writer) Writer.Error!void { }; } -/// Print the string as escaped contents of a single-quoted string. -pub fn charEscape(bytes: []const u8, w: *Writer) Writer.Error!void { - for (bytes) |byte| switch (byte) { +/// Print as escaped contents of a single-quoted string. +pub fn charEscape(codepoint: u21, w: *Writer) Writer.Error!void { + switch (codepoint) { '\n' => try w.writeAll("\\n"), '\r' => try w.writeAll("\\r"), '\t' => try w.writeAll("\\t"), '\\' => try w.writeAll("\\\\"), - '"' => try w.writeByte('"'), '\'' => try w.writeAll("\\'"), - ' ', '!', '#'...'&', '('...'[', ']'...'~' => try w.writeByte(byte), + '"', ' ', '!', '#'...'&', '('...'[', ']'...'~' => try w.writeByte(@intCast(codepoint)), else => { - try w.writeAll("\\x"); - try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' }); + if (std.math.cast(u8, codepoint)) |byte| { + try w.writeAll("\\x"); + try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' }); + } else { + try w.writeAll("\\u{"); + try w.printInt(codepoint, 16, .lower, .{}); + try w.writeByte('}'); + } }, - }; + } } pub fn isValidId(bytes: []const u8) bool { diff --git a/lib/std/zig/Ast.zig b/lib/std/zig/Ast.zig index 1f36c0fdbf6040d2417aaa35ee9884f393daf3c5..c15693fd625ee4cd521326cfcc92a031f7dab312 100644 --- a/lib/std/zig/Ast.zig +++ b/lib/std/zig/Ast.zig @@ -574,7 +574,7 @@ pub fn renderError(tree: Ast, parse_error: Error, w: *Writer) Writer.Error!void '/' => "comment", else => unreachable, }, - std.zig.fmtChar(tok_slice[parse_error.extra.offset..][0..1]), + std.zig.fmtChar(tok_slice[parse_error.extra.offset]), }); }, diff --git a/lib/std/zon/parse.zig b/lib/std/zon/parse.zig index d5df5693ec13f7abb3a9f3f53836a0a50a5c6c92..96a7fa65953320b04310d647c6806fec49971ca5 100644 --- a/lib/std/zon/parse.zig +++ b/lib/std/zon/parse.zig @@ -64,14 +64,14 @@ pub const Error = union(enum) { } }; - fn formatMessage(self: []const u8, w: *std.io.Writer) std.io.Writer.Error!void { + fn formatMessage(self: []const u8, w: *std.Io.Writer) std.Io.Writer.Error!void { // Just writes the string for now, but we're keeping this behind a formatter so we have // the option to extend it in the future to print more advanced messages (like `Error` // does) without breaking the API. try w.writeAll(self); } - pub fn fmtMessage(self: Note, diag: *const Diagnostics) std.fmt.Formatter([]const u8, Note.formatMessage) { + pub fn fmtMessage(self: Note, diag: *const Diagnostics) std.fmt.Alt([]const u8, Note.formatMessage) { return .{ .data = switch (self) { .zoir => |note| note.msg.get(diag.zoir), .type_check => |note| note.msg, @@ -147,14 +147,14 @@ pub const Error = union(enum) { diag: *const Diagnostics, }; - fn formatMessage(self: FormatMessage, w: *std.io.Writer) std.io.Writer.Error!void { + fn formatMessage(self: FormatMessage, w: *std.Io.Writer) std.Io.Writer.Error!void { switch (self.err) { .zoir => |err| try w.writeAll(err.msg.get(self.diag.zoir)), .type_check => |tc| try w.writeAll(tc.message), } } - pub fn fmtMessage(self: @This(), diag: *const Diagnostics) std.fmt.Formatter(FormatMessage, formatMessage) { + pub fn fmtMessage(self: @This(), diag: *const Diagnostics) std.fmt.Alt(FormatMessage, formatMessage) { return .{ .data = .{ .err = self, .diag = diag, @@ -226,7 +226,7 @@ pub const Diagnostics = struct { return .{ .diag = self }; } - pub fn format(self: *const @This(), w: *std.io.Writer) std.io.Writer.Error!void { + pub fn format(self: *const @This(), w: *std.Io.Writer) std.Io.Writer.Error!void { var errors = self.iterateErrors(); while (errors.next()) |err| { const loc = err.getLocation(self); @@ -606,7 +606,7 @@ const Parser = struct { } } - fn parseSlicePointer(self: *@This(), T: type, node: Zoir.Node.Index) !T { + fn parseSlicePointer(self: *@This(), T: type, node: Zoir.Node.Index) ParseExprInnerError!T { switch (node.get(self.zoir)) { .string_literal => return self.parseString(T, node), .array_literal => |nodes| return self.parseSlice(T, nodes), @@ -1048,6 +1048,7 @@ const Parser = struct { name: []const u8, ) error{ OutOfMemory, ParseZon } { @branchHint(.cold); + const gpa = self.gpa; const token = if (field) |f| b: { var buf: [2]Ast.Node.Index = undefined; const struct_init = self.ast.fullStructInit(&buf, node.getAstNode(self.zoir)).?; @@ -1065,13 +1066,12 @@ const Parser = struct { }; } else b: { const msg = "supported: "; - var buf: std.ArrayListUnmanaged(u8) = try .initCapacity(self.gpa, 64); - defer buf.deinit(self.gpa); - const writer = buf.writer(self.gpa); - try writer.writeAll(msg); + var buf: std.ArrayListUnmanaged(u8) = try .initCapacity(gpa, 64); + defer buf.deinit(gpa); + try buf.appendSlice(gpa, msg); inline for (info.fields, 0..) |field_info, i| { - if (i != 0) try writer.writeAll(", "); - try writer.print("'{f}'", .{std.zig.fmtIdFlags(field_info.name, .{ + if (i != 0) try buf.appendSlice(gpa, ", "); + try buf.print(gpa, "'{f}'", .{std.zig.fmtIdFlags(field_info.name, .{ .allow_primitive = true, .allow_underscore = true, })}); @@ -1079,7 +1079,7 @@ const Parser = struct { break :b .{ .token = token, .offset = 0, - .msg = try buf.toOwnedSlice(self.gpa), + .msg = try buf.toOwnedSlice(gpa), .owned = true, }; }; diff --git a/lib/std/zon/stringify.zig b/lib/std/zon/stringify.zig index 1a38dc75795fa9606718e321728432ec1eade4eb..4cb2640063aa8870efed3abf21fdf701cb7ca9b8 100644 --- a/lib/std/zon/stringify.zig +++ b/lib/std/zon/stringify.zig @@ -22,6 +22,7 @@ const std = @import("std"); const assert = std.debug.assert; +const Writer = std.Io.Writer; /// Options for `serialize`. pub const SerializeOptions = struct { @@ -40,15 +41,12 @@ pub const SerializeOptions = struct { /// Serialize the given value as ZON. /// /// It is asserted at comptime that `@TypeOf(val)` is not a recursive type. -pub fn serialize( - val: anytype, - options: SerializeOptions, - writer: anytype, -) @TypeOf(writer).Error!void { - var sz = serializer(writer, .{ - .whitespace = options.whitespace, - }); - try sz.value(val, .{ +pub fn serialize(val: anytype, options: SerializeOptions, writer: *Writer) Writer.Error!void { + var s: Serializer = .{ + .writer = writer, + .options = .{ .whitespace = options.whitespace }, + }; + try s.value(val, .{ .emit_codepoint_literals = options.emit_codepoint_literals, .emit_strings_as_containers = options.emit_strings_as_containers, .emit_default_optional_fields = options.emit_default_optional_fields, @@ -62,13 +60,14 @@ pub fn serialize( pub fn serializeMaxDepth( val: anytype, options: SerializeOptions, - writer: anytype, + writer: *Writer, depth: usize, -) (@TypeOf(writer).Error || error{ExceededMaxDepth})!void { - var sz = serializer(writer, .{ - .whitespace = options.whitespace, - }); - try sz.valueMaxDepth(val, .{ +) Serializer.DepthError!void { + var s: Serializer = .{ + .writer = writer, + .options = .{ .whitespace = options.whitespace }, + }; + try s.valueMaxDepth(val, .{ .emit_codepoint_literals = options.emit_codepoint_literals, .emit_strings_as_containers = options.emit_strings_as_containers, .emit_default_optional_fields = options.emit_default_optional_fields, @@ -81,44 +80,45 @@ pub fn serializeMaxDepth( pub fn serializeArbitraryDepth( val: anytype, options: SerializeOptions, - writer: anytype, -) @TypeOf(writer).Error!void { - var sz = serializer(writer, .{ - .whitespace = options.whitespace, - }); - try sz.valueArbitraryDepth(val, .{ + writer: *Writer, +) Serializer.Error!void { + var s: Serializer = .{ + .writer = writer, + .options = .{ .whitespace = options.whitespace }, + }; + try s.valueArbitraryDepth(val, .{ .emit_codepoint_literals = options.emit_codepoint_literals, .emit_strings_as_containers = options.emit_strings_as_containers, .emit_default_optional_fields = options.emit_default_optional_fields, }); } -fn typeIsRecursive(comptime T: type) bool { - return comptime typeIsRecursiveImpl(T, &.{}); +inline fn typeIsRecursive(comptime T: type) bool { + return comptime typeIsRecursiveInner(T, &.{}); } -fn typeIsRecursiveImpl(comptime T: type, comptime prev_visited: []const type) bool { +fn typeIsRecursiveInner(comptime T: type, comptime prev_visited: []const type) bool { for (prev_visited) |V| { if (V == T) return true; } const visited = prev_visited ++ .{T}; return switch (@typeInfo(T)) { - .pointer => |pointer| typeIsRecursiveImpl(pointer.child, visited), - .optional => |optional| typeIsRecursiveImpl(optional.child, visited), - .array => |array| typeIsRecursiveImpl(array.child, visited), - .vector => |vector| typeIsRecursiveImpl(vector.child, visited), + .pointer => |pointer| typeIsRecursiveInner(pointer.child, visited), + .optional => |optional| typeIsRecursiveInner(optional.child, visited), + .array => |array| typeIsRecursiveInner(array.child, visited), + .vector => |vector| typeIsRecursiveInner(vector.child, visited), .@"struct" => |@"struct"| for (@"struct".fields) |field| { - if (typeIsRecursiveImpl(field.type, visited)) break true; + if (typeIsRecursiveInner(field.type, visited)) break true; } else false, .@"union" => |@"union"| inline for (@"union".fields) |field| { - if (typeIsRecursiveImpl(field.type, visited)) break true; + if (typeIsRecursiveInner(field.type, visited)) break true; } else false, else => false, }; } -fn canSerializeType(T: type) bool { +inline fn canSerializeType(T: type) bool { comptime return canSerializeTypeInner(T, &.{}, false); } @@ -343,12 +343,6 @@ test "std.zon checkValueDepth" { try expectValueDepthEquals(3, @as([]const []const u8, &.{&.{ 1, 2, 3 }})); } -/// Options for `Serializer`. -pub const SerializerOptions = struct { - /// If false, only syntactically necessary whitespace is emitted. - whitespace: bool = true, -}; - /// Determines when to emit Unicode code point literals as opposed to integer literals. pub const EmitCodepointLiterals = enum { /// Never emit Unicode code point literals. @@ -440,634 +434,616 @@ pub const SerializeContainerOptions = struct { /// For manual serialization of containers, see: /// * `beginStruct` /// * `beginTuple` -/// -/// # Example -/// ```zig -/// var sz = serializer(writer, .{}); -/// var vec2 = try sz.beginStruct(.{}); -/// try vec2.field("x", 1.5, .{}); -/// try vec2.fieldPrefix(); -/// try sz.value(2.5); -/// try vec2.end(); -/// ``` -pub fn Serializer(Writer: type) type { - return struct { - const Self = @This(); +pub const Serializer = struct { + options: Options = .{}, + indent_level: u8 = 0, + writer: *Writer, - options: SerializerOptions, - indent_level: u8, - writer: Writer, + pub const Error = Writer.Error; + pub const DepthError = Error || error{ExceededMaxDepth}; - /// Initialize a serializer. - fn init(writer: Writer, options: SerializerOptions) Self { - return .{ - .options = options, - .writer = writer, - .indent_level = 0, - }; - } + pub const Options = struct { + /// If false, only syntactically necessary whitespace is emitted. + whitespace: bool = true, + }; - /// Serialize a value, similar to `serialize`. - pub fn value(self: *Self, val: anytype, options: ValueOptions) Writer.Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - return self.valueArbitraryDepth(val, options); - } + /// Serialize a value, similar to `serialize`. + pub fn value(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + return self.valueArbitraryDepth(val, options); + } - /// Serialize a value, similar to `serializeMaxDepth`. - pub fn valueMaxDepth( - self: *Self, - val: anytype, - options: ValueOptions, - depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try checkValueDepth(val, depth); - return self.valueArbitraryDepth(val, options); - } + /// Serialize a value, similar to `serializeMaxDepth`. + /// Can return `error.ExceededMaxDepth`. + pub fn valueMaxDepth(self: *Serializer, val: anytype, options: ValueOptions, depth: usize) DepthError!void { + try checkValueDepth(val, depth); + return self.valueArbitraryDepth(val, options); + } - /// Serialize a value, similar to `serializeArbitraryDepth`. - pub fn valueArbitraryDepth( - self: *Self, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - comptime assert(canSerializeType(@TypeOf(val))); - switch (@typeInfo(@TypeOf(val))) { - .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { - self.codePoint(c) catch |err| switch (err) { - error.InvalidCodepoint => unreachable, // Already validated - else => |e| return e, - }; - } else { - try self.int(val); - }, - .float, .comptime_float => try self.float(val), - .bool, .null => try std.fmt.format(self.writer, "{}", .{val}), - .enum_literal => try self.ident(@tagName(val)), - .@"enum" => try self.ident(@tagName(val)), - .pointer => |pointer| { - // Try to serialize as a string - const item: ?type = switch (@typeInfo(pointer.child)) { - .array => |array| array.child, - else => if (pointer.size == .slice) pointer.child else null, - }; - if (item == u8 and - (pointer.sentinel() == null or pointer.sentinel() == 0) and - !options.emit_strings_as_containers) - { - return try self.string(val); - } + /// Serialize a value, similar to `serializeArbitraryDepth`. + pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(canSerializeType(@TypeOf(val))); + switch (@typeInfo(@TypeOf(val))) { + .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { + self.codePoint(c) catch |err| switch (err) { + error.InvalidCodepoint => unreachable, // Already validated + else => |e| return e, + }; + } else { + try self.int(val); + }, + .float, .comptime_float => try self.float(val), + .bool, .null => try self.writer.print("{}", .{val}), + .enum_literal => try self.ident(@tagName(val)), + .@"enum" => try self.ident(@tagName(val)), + .pointer => |pointer| { + // Try to serialize as a string + const item: ?type = switch (@typeInfo(pointer.child)) { + .array => |array| array.child, + else => if (pointer.size == .slice) pointer.child else null, + }; + if (item == u8 and + (pointer.sentinel() == null or pointer.sentinel() == 0) and + !options.emit_strings_as_containers) + { + return try self.string(val); + } - // Serialize as either a tuple or as the child type - switch (pointer.size) { - .slice => try self.tupleImpl(val, options), - .one => try self.valueArbitraryDepth(val.*, options), - else => comptime unreachable, - } - }, - .array => { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = val.len } }, - ); - for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - .@"struct" => |@"struct"| if (@"struct".is_tuple) { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = @"struct".fields.len } }, - ); - inline for (val) |field_value| { - try container.fieldArbitraryDepth(field_value, options); - } - try container.end(); - } else { - // Decide which fields to emit - const fields, const skipped: [@"struct".fields.len]bool = if (options.emit_default_optional_fields) b: { - break :b .{ @"struct".fields.len, @splat(false) }; - } else b: { - var fields = @"struct".fields.len; - var skipped: [@"struct".fields.len]bool = @splat(false); - inline for (@"struct".fields, &skipped) |field_info, *skip| { - if (field_info.default_value_ptr) |ptr| { - const default: *const field_info.type = @ptrCast(@alignCast(ptr)); - const field_value = @field(val, field_info.name); - if (std.meta.eql(field_value, default.*)) { - skip.* = true; - fields -= 1; - } + // Serialize as either a tuple or as the child type + switch (pointer.size) { + .slice => try self.tupleImpl(val, options), + .one => try self.valueArbitraryDepth(val.*, options), + else => comptime unreachable, + } + }, + .array => { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = val.len } }, + ); + for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + .@"struct" => |@"struct"| if (@"struct".is_tuple) { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = @"struct".fields.len } }, + ); + inline for (val) |field_value| { + try container.fieldArbitraryDepth(field_value, options); + } + try container.end(); + } else { + // Decide which fields to emit + const fields, const skipped: [@"struct".fields.len]bool = if (options.emit_default_optional_fields) b: { + break :b .{ @"struct".fields.len, @splat(false) }; + } else b: { + var fields = @"struct".fields.len; + var skipped: [@"struct".fields.len]bool = @splat(false); + inline for (@"struct".fields, &skipped) |field_info, *skip| { + if (field_info.default_value_ptr) |ptr| { + const default: *const field_info.type = @ptrCast(@alignCast(ptr)); + const field_value = @field(val, field_info.name); + if (std.meta.eql(field_value, default.*)) { + skip.* = true; + fields -= 1; } } - break :b .{ fields, skipped }; - }; - - // Emit those fields - var container = try self.beginStruct( - .{ .whitespace_style = .{ .fields = fields } }, - ); - inline for (@"struct".fields, skipped) |field_info, skip| { - if (!skip) { - try container.fieldArbitraryDepth( - field_info.name, - @field(val, field_info.name), - options, - ); - } } - try container.end(); - }, - .@"union" => |@"union"| { - comptime assert(@"union".tag_type != null); - switch (val) { - inline else => |pl, tag| if (@TypeOf(pl) == void) - try self.writer.print(".{s}", .{@tagName(tag)}) - else { - var container = try self.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); - - try container.fieldArbitraryDepth( - @tagName(tag), - pl, - options, - ); - - try container.end(); - }, - } - }, - .optional => if (val) |inner| { - try self.valueArbitraryDepth(inner, options); - } else { - try self.writer.writeAll("null"); - }, - .vector => |vector| { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = vector.len } }, - ); - for (0..vector.len) |i| { - try container.fieldArbitraryDepth(val[i], options); + break :b .{ fields, skipped }; + }; + + // Emit those fields + var container = try self.beginStruct( + .{ .whitespace_style = .{ .fields = fields } }, + ); + inline for (@"struct".fields, skipped) |field_info, skip| { + if (!skip) { + try container.fieldArbitraryDepth( + field_info.name, + @field(val, field_info.name), + options, + ); } - try container.end(); - }, + } + try container.end(); + }, + .@"union" => |@"union"| { + comptime assert(@"union".tag_type != null); + switch (val) { + inline else => |pl, tag| if (@TypeOf(pl) == void) + try self.writer.print(".{s}", .{@tagName(tag)}) + else { + var container = try self.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); + + try container.fieldArbitraryDepth( + @tagName(tag), + pl, + options, + ); + + try container.end(); + }, + } + }, + .optional => if (val) |inner| { + try self.valueArbitraryDepth(inner, options); + } else { + try self.writer.writeAll("null"); + }, + .vector => |vector| { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = vector.len } }, + ); + for (0..vector.len) |i| { + try container.fieldArbitraryDepth(val[i], options); + } + try container.end(); + }, - else => comptime unreachable, + else => comptime unreachable, + } + } + + /// Serialize an integer. + pub fn int(self: *Serializer, val: anytype) Error!void { + try self.writer.printInt(val, 10, .lower, .{}); + } + + /// Serialize a float. + pub fn float(self: *Serializer, val: anytype) Error!void { + switch (@typeInfo(@TypeOf(val))) { + .float => if (std.math.isNan(val)) { + return self.writer.writeAll("nan"); + } else if (std.math.isPositiveInf(val)) { + return self.writer.writeAll("inf"); + } else if (std.math.isNegativeInf(val)) { + return self.writer.writeAll("-inf"); + } else if (std.math.isNegativeZero(val)) { + return self.writer.writeAll("-0.0"); + } else { + try self.writer.print("{d}", .{val}); + }, + .comptime_float => if (val == 0) { + return self.writer.writeAll("0"); + } else { + try self.writer.print("{d}", .{val}); + }, + else => comptime unreachable, + } + } + + /// Serialize `name` as an identifier prefixed with `.`. + /// + /// Escapes the identifier if necessary. + pub fn ident(self: *Serializer, name: []const u8) Error!void { + try self.writer.print(".{f}", .{std.zig.fmtIdPU(name)}); + } + + pub const CodePointError = Error || error{InvalidCodepoint}; + + /// Serialize `val` as a Unicode codepoint. + /// + /// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint. + pub fn codePoint(self: *Serializer, val: u21) CodePointError!void { + try self.writer.print("'{f}'", .{std.zig.fmtChar(val)}); + } + + /// Like `value`, but always serializes `val` as a tuple. + /// + /// Will fail at comptime if `val` is not a tuple, array, pointer to an array, or slice. + pub fn tuple(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + try self.tupleArbitraryDepth(val, options); + } + + /// Like `tuple`, but recursive types are allowed. + /// + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + pub fn tupleMaxDepth( + self: *Serializer, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try checkValueDepth(val, depth); + try self.tupleArbitraryDepth(val, options); + } + + /// Like `tuple`, but recursive types are allowed. + /// + /// It is the caller's responsibility to ensure that `val` does not contain cycles. + pub fn tupleArbitraryDepth( + self: *Serializer, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.tupleImpl(val, options); + } + + fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(canSerializeType(@TypeOf(val))); + switch (@typeInfo(@TypeOf(val))) { + .@"struct" => { + var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); + inline for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + .pointer, .array => { + var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); + for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + else => comptime unreachable, + } + } + + /// Like `value`, but always serializes `val` as a string. + pub fn string(self: *Serializer, val: []const u8) Error!void { + try self.writer.print("\"{f}\"", .{std.zig.fmtString(val)}); + } + + /// Options for formatting multiline strings. + pub const MultilineStringOptions = struct { + /// If top level is true, whitespace before and after the multiline string is elided. + /// If it is true, a newline is printed, then the value, followed by a newline, and if + /// whitespace is true any necessary indentation follows. + top_level: bool = false, + }; + + pub const MultilineStringError = Error || error{InnerCarriageReturn}; + + /// Like `value`, but always serializes to a multiline string literal. + /// + /// Returns `error.InnerCarriageReturn` if `val` contains a CR not followed by a newline, + /// since multiline strings cannot represent CR without a following newline. + pub fn multilineString( + self: *Serializer, + val: []const u8, + options: MultilineStringOptions, + ) MultilineStringError!void { + // Make sure the string does not contain any carriage returns not followed by a newline + var i: usize = 0; + while (i < val.len) : (i += 1) { + if (val[i] == '\r') { + if (i + 1 < val.len) { + if (val[i + 1] == '\n') { + i += 1; + continue; + } + } + return error.InnerCarriageReturn; } } - /// Serialize an integer. - pub fn int(self: *Self, val: anytype) Writer.Error!void { - //try self.writer.printInt(val, 10, .lower, .{}); - try std.fmt.format(self.writer, "{d}", .{val}); + if (!options.top_level) { + try self.newline(); + try self.indent(); } - /// Serialize a float. - pub fn float(self: *Self, val: anytype) Writer.Error!void { - switch (@typeInfo(@TypeOf(val))) { - .float => if (std.math.isNan(val)) { - return self.writer.writeAll("nan"); - } else if (std.math.isPositiveInf(val)) { - return self.writer.writeAll("inf"); - } else if (std.math.isNegativeInf(val)) { - return self.writer.writeAll("-inf"); - } else if (std.math.isNegativeZero(val)) { - return self.writer.writeAll("-0.0"); - } else { - try std.fmt.format(self.writer, "{d}", .{val}); - }, - .comptime_float => if (val == 0) { - return self.writer.writeAll("0"); - } else { - try std.fmt.format(self.writer, "{d}", .{val}); - }, - else => comptime unreachable, + try self.writer.writeAll("\\\\"); + for (val) |c| { + if (c != '\r') { + try self.writer.writeByte(c); // We write newlines here even if whitespace off + if (c == '\n') { + try self.indent(); + try self.writer.writeAll("\\\\"); + } } } - /// Serialize `name` as an identifier prefixed with `.`. - /// - /// Escapes the identifier if necessary. - pub fn ident(self: *Self, name: []const u8) Writer.Error!void { - try self.writer.print(".{f}", .{std.zig.fmtIdPU(name)}); + if (!options.top_level) { + try self.writer.writeByte('\n'); // Even if whitespace off + try self.indent(); + } + } + + /// Create a `Struct` for writing ZON structs field by field. + pub fn beginStruct( + self: *Serializer, + options: SerializeContainerOptions, + ) Error!Struct { + return Struct.begin(self, options); + } + + /// Creates a `Tuple` for writing ZON tuples field by field. + pub fn beginTuple( + self: *Serializer, + options: SerializeContainerOptions, + ) Error!Tuple { + return Tuple.begin(self, options); + } + + fn indent(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.splatByteAll(' ', 4 * self.indent_level); } + } - /// Serialize `val` as a Unicode codepoint. - /// - /// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint. - pub fn codePoint( - self: *Self, - val: u21, - ) (Writer.Error || error{InvalidCodepoint})!void { - var buf: [8]u8 = undefined; - const len = std.unicode.utf8Encode(val, &buf) catch return error.InvalidCodepoint; - const str = buf[0..len]; - try std.fmt.format(self.writer, "'{f}'", .{std.zig.fmtChar(str)}); + fn newline(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.writeByte('\n'); } + } - /// Like `value`, but always serializes `val` as a tuple. - /// - /// Will fail at comptime if `val` is not a tuple, array, pointer to an array, or slice. - pub fn tuple(self: *Self, val: anytype, options: ValueOptions) Writer.Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - try self.tupleArbitraryDepth(val, options); + fn newlineOrSpace(self: *Serializer, len: usize) Error!void { + if (self.containerShouldWrap(len)) { + try self.newline(); + } else { + try self.space(); + } + } + + fn space(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.writeByte(' '); + } + } + + /// Writes ZON tuples field by field. + pub const Tuple = struct { + container: Container, + + fn begin(parent: *Serializer, options: SerializeContainerOptions) Error!Tuple { + return .{ + .container = try Container.begin(parent, .anon, options), + }; } - /// Like `tuple`, but recursive types are allowed. + /// Finishes serializing the tuple. /// + /// Prints a trailing comma as configured when appropriate, and the closing bracket. + pub fn end(self: *Tuple) Error!void { + try self.container.end(); + self.* = undefined; + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. + pub fn field( + self: *Tuple, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.field(null, val, options); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - pub fn tupleMaxDepth( - self: *Self, + pub fn fieldMaxDepth( + self: *Tuple, val: anytype, options: ValueOptions, depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try checkValueDepth(val, depth); - try self.tupleArbitraryDepth(val, options); + ) DepthError!void { + try self.container.fieldMaxDepth(null, val, options, depth); } - /// Like `tuple`, but recursive types are allowed. - /// - /// It is the caller's responsibility to ensure that `val` does not contain cycles. - pub fn tupleArbitraryDepth( - self: *Self, + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by + /// `valueArbitraryDepth`. + pub fn fieldArbitraryDepth( + self: *Tuple, val: anytype, options: ValueOptions, - ) Writer.Error!void { - try self.tupleImpl(val, options); + ) Error!void { + try self.container.fieldArbitraryDepth(null, val, options); } - fn tupleImpl(self: *Self, val: anytype, options: ValueOptions) Writer.Error!void { - comptime assert(canSerializeType(@TypeOf(val))); - switch (@typeInfo(@TypeOf(val))) { - .@"struct" => { - var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); - inline for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - .pointer, .array => { - var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); - for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - else => comptime unreachable, - } + /// Starts a field with a struct as a value. Returns the struct. + pub fn beginStructField( + self: *Tuple, + options: SerializeContainerOptions, + ) Error!Struct { + try self.fieldPrefix(); + return self.container.serializer.beginStruct(options); } - /// Like `value`, but always serializes `val` as a string. - pub fn string(self: *Self, val: []const u8) Writer.Error!void { - try std.fmt.format(self.writer, "\"{f}\"", .{std.zig.fmtString(val)}); + /// Starts a field with a tuple as a value. Returns the tuple. + pub fn beginTupleField( + self: *Tuple, + options: SerializeContainerOptions, + ) Error!Tuple { + try self.fieldPrefix(); + return self.container.serializer.beginTuple(options); } - /// Options for formatting multiline strings. - pub const MultilineStringOptions = struct { - /// If top level is true, whitespace before and after the multiline string is elided. - /// If it is true, a newline is printed, then the value, followed by a newline, and if - /// whitespace is true any necessary indentation follows. - top_level: bool = false, - }; - - /// Like `value`, but always serializes to a multiline string literal. - /// - /// Returns `error.InnerCarriageReturn` if `val` contains a CR not followed by a newline, - /// since multiline strings cannot represent CR without a following newline. - pub fn multilineString( - self: *Self, - val: []const u8, - options: MultilineStringOptions, - ) (Writer.Error || error{InnerCarriageReturn})!void { - // Make sure the string does not contain any carriage returns not followed by a newline - var i: usize = 0; - while (i < val.len) : (i += 1) { - if (val[i] == '\r') { - if (i + 1 < val.len) { - if (val[i + 1] == '\n') { - i += 1; - continue; - } - } - return error.InnerCarriageReturn; - } - } + /// Print a field prefix. This prints any necessary commas, and whitespace as + /// configured. Useful if you want to serialize the field value yourself. + pub fn fieldPrefix(self: *Tuple) Error!void { + try self.container.fieldPrefix(null); + } + }; - if (!options.top_level) { - try self.newline(); - try self.indent(); - } + /// Writes ZON structs field by field. + pub const Struct = struct { + container: Container, - try self.writer.writeAll("\\\\"); - for (val) |c| { - if (c != '\r') { - try self.writer.writeByte(c); // We write newlines here even if whitespace off - if (c == '\n') { - try self.indent(); - try self.writer.writeAll("\\\\"); - } - } - } - - if (!options.top_level) { - try self.writer.writeByte('\n'); // Even if whitespace off - try self.indent(); - } + fn begin(parent: *Serializer, options: SerializeContainerOptions) Error!Struct { + return .{ + .container = try Container.begin(parent, .named, options), + }; } - /// Create a `Struct` for writing ZON structs field by field. - pub fn beginStruct( - self: *Self, - options: SerializeContainerOptions, - ) Writer.Error!Struct { - return Struct.begin(self, options); + /// Finishes serializing the struct. + /// + /// Prints a trailing comma as configured when appropriate, and the closing bracket. + pub fn end(self: *Struct) Error!void { + try self.container.end(); + self.* = undefined; } - /// Creates a `Tuple` for writing ZON tuples field by field. - pub fn beginTuple( - self: *Self, - options: SerializeContainerOptions, - ) Writer.Error!Tuple { - return Tuple.begin(self, options); + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. + pub fn field( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.field(name, val, options); } - fn indent(self: *Self) Writer.Error!void { - if (self.options.whitespace) { - try self.writer.writeByteNTimes(' ', 4 * self.indent_level); - } + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + pub fn fieldMaxDepth( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try self.container.fieldMaxDepth(name, val, options, depth); } - fn newline(self: *Self) Writer.Error!void { - if (self.options.whitespace) { - try self.writer.writeByte('\n'); - } + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by + /// `valueArbitraryDepth`. + pub fn fieldArbitraryDepth( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.fieldArbitraryDepth(name, val, options); } - fn newlineOrSpace(self: *Self, len: usize) Writer.Error!void { - if (self.containerShouldWrap(len)) { - try self.newline(); - } else { - try self.space(); - } + /// Starts a field with a struct as a value. Returns the struct. + pub fn beginStructField( + self: *Struct, + name: []const u8, + options: SerializeContainerOptions, + ) Error!Struct { + try self.fieldPrefix(name); + return self.container.serializer.beginStruct(options); } - fn space(self: *Self) Writer.Error!void { - if (self.options.whitespace) { - try self.writer.writeByte(' '); - } + /// Starts a field with a tuple as a value. Returns the tuple. + pub fn beginTupleField( + self: *Struct, + name: []const u8, + options: SerializeContainerOptions, + ) Error!Tuple { + try self.fieldPrefix(name); + return self.container.serializer.beginTuple(options); } - /// Writes ZON tuples field by field. - pub const Tuple = struct { - container: Container, - - fn begin(parent: *Self, options: SerializeContainerOptions) Writer.Error!Tuple { - return .{ - .container = try Container.begin(parent, .anon, options), - }; - } - - /// Finishes serializing the tuple. - /// - /// Prints a trailing comma as configured when appropriate, and the closing bracket. - pub fn end(self: *Tuple) Writer.Error!void { - try self.container.end(); - self.* = undefined; - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. - pub fn field( - self: *Tuple, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.container.field(null, val, options); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. - pub fn fieldMaxDepth( - self: *Tuple, - val: anytype, - options: ValueOptions, - depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try self.container.fieldMaxDepth(null, val, options, depth); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by - /// `valueArbitraryDepth`. - pub fn fieldArbitraryDepth( - self: *Tuple, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.container.fieldArbitraryDepth(null, val, options); - } - - /// Starts a field with a struct as a value. Returns the struct. - pub fn beginStructField( - self: *Tuple, - options: SerializeContainerOptions, - ) Writer.Error!Struct { - try self.fieldPrefix(); - return self.container.serializer.beginStruct(options); - } - - /// Starts a field with a tuple as a value. Returns the tuple. - pub fn beginTupleField( - self: *Tuple, - options: SerializeContainerOptions, - ) Writer.Error!Tuple { - try self.fieldPrefix(); - return self.container.serializer.beginTuple(options); - } - - /// Print a field prefix. This prints any necessary commas, and whitespace as - /// configured. Useful if you want to serialize the field value yourself. - pub fn fieldPrefix(self: *Tuple) Writer.Error!void { - try self.container.fieldPrefix(null); - } - }; - - /// Writes ZON structs field by field. - pub const Struct = struct { - container: Container, - - fn begin(parent: *Self, options: SerializeContainerOptions) Writer.Error!Struct { - return .{ - .container = try Container.begin(parent, .named, options), - }; - } - - /// Finishes serializing the struct. - /// - /// Prints a trailing comma as configured when appropriate, and the closing bracket. - pub fn end(self: *Struct) Writer.Error!void { - try self.container.end(); - self.* = undefined; - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. - pub fn field( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.container.field(name, val, options); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. - pub fn fieldMaxDepth( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try self.container.fieldMaxDepth(name, val, options, depth); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by - /// `valueArbitraryDepth`. - pub fn fieldArbitraryDepth( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.container.fieldArbitraryDepth(name, val, options); - } - - /// Starts a field with a struct as a value. Returns the struct. - pub fn beginStructField( - self: *Struct, - name: []const u8, - options: SerializeContainerOptions, - ) Writer.Error!Struct { - try self.fieldPrefix(name); - return self.container.serializer.beginStruct(options); - } - - /// Starts a field with a tuple as a value. Returns the tuple. - pub fn beginTupleField( - self: *Struct, - name: []const u8, - options: SerializeContainerOptions, - ) Writer.Error!Tuple { - try self.fieldPrefix(name); - return self.container.serializer.beginTuple(options); - } + /// Print a field prefix. This prints any necessary commas, the field name (escaped if + /// necessary) and whitespace as configured. Useful if you want to serialize the field + /// value yourself. + pub fn fieldPrefix(self: *Struct, name: []const u8) Error!void { + try self.container.fieldPrefix(name); + } + }; - /// Print a field prefix. This prints any necessary commas, the field name (escaped if - /// necessary) and whitespace as configured. Useful if you want to serialize the field - /// value yourself. - pub fn fieldPrefix(self: *Struct, name: []const u8) Writer.Error!void { - try self.container.fieldPrefix(name); - } - }; + const Container = struct { + const FieldStyle = enum { named, anon }; - const Container = struct { - const FieldStyle = enum { named, anon }; + serializer: *Serializer, + field_style: FieldStyle, + options: SerializeContainerOptions, + empty: bool, - serializer: *Self, + fn begin( + sz: *Serializer, field_style: FieldStyle, options: SerializeContainerOptions, - empty: bool, + ) Error!Container { + if (options.shouldWrap()) sz.indent_level +|= 1; + try sz.writer.writeAll(".{"); + return .{ + .serializer = sz, + .field_style = field_style, + .options = options, + .empty = true, + }; + } - fn begin( - sz: *Self, - field_style: FieldStyle, - options: SerializeContainerOptions, - ) Writer.Error!Container { - if (options.shouldWrap()) sz.indent_level +|= 1; - try sz.writer.writeAll(".{"); - return .{ - .serializer = sz, - .field_style = field_style, - .options = options, - .empty = true, - }; - } - - fn end(self: *Container) Writer.Error!void { - if (self.options.shouldWrap()) self.serializer.indent_level -|= 1; - if (!self.empty) { - if (self.options.shouldWrap()) { - if (self.serializer.options.whitespace) { - try self.serializer.writer.writeByte(','); - } - try self.serializer.newline(); - try self.serializer.indent(); - } else if (!self.shouldElideSpaces()) { - try self.serializer.space(); - } - } - try self.serializer.writer.writeByte('}'); - self.* = undefined; - } - - fn fieldPrefix(self: *Container, name: ?[]const u8) Writer.Error!void { - if (!self.empty) { - try self.serializer.writer.writeByte(','); - } - self.empty = false; + fn end(self: *Container) Error!void { + if (self.options.shouldWrap()) self.serializer.indent_level -|= 1; + if (!self.empty) { if (self.options.shouldWrap()) { + if (self.serializer.options.whitespace) { + try self.serializer.writer.writeByte(','); + } try self.serializer.newline(); + try self.serializer.indent(); } else if (!self.shouldElideSpaces()) { try self.serializer.space(); } - if (self.options.shouldWrap()) try self.serializer.indent(); - if (name) |n| { - try self.serializer.ident(n); - try self.serializer.space(); - try self.serializer.writer.writeByte('='); - try self.serializer.space(); - } } + try self.serializer.writer.writeByte('}'); + self.* = undefined; + } - fn field( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - try self.fieldArbitraryDepth(name, val, options); + fn fieldPrefix(self: *Container, name: ?[]const u8) Error!void { + if (!self.empty) { + try self.serializer.writer.writeByte(','); } - - fn fieldMaxDepth( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try checkValueDepth(val, depth); - try self.fieldArbitraryDepth(name, val, options); + self.empty = false; + if (self.options.shouldWrap()) { + try self.serializer.newline(); + } else if (!self.shouldElideSpaces()) { + try self.serializer.space(); } - - fn fieldArbitraryDepth( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.fieldPrefix(name); - try self.serializer.valueArbitraryDepth(val, options); + if (self.options.shouldWrap()) try self.serializer.indent(); + if (name) |n| { + try self.serializer.ident(n); + try self.serializer.space(); + try self.serializer.writer.writeByte('='); + try self.serializer.space(); } + } - fn shouldElideSpaces(self: *const Container) bool { - return switch (self.options.whitespace_style) { - .fields => |fields| self.field_style != .named and fields == 1, - else => false, - }; - } - }; + fn field( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + try self.fieldArbitraryDepth(name, val, options); + } + + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + fn fieldMaxDepth( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try checkValueDepth(val, depth); + try self.fieldArbitraryDepth(name, val, options); + } + + fn fieldArbitraryDepth( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.fieldPrefix(name); + try self.serializer.valueArbitraryDepth(val, options); + } + + fn shouldElideSpaces(self: *const Container) bool { + return switch (self.options.whitespace_style) { + .fields => |fields| self.field_style != .named and fields == 1, + else => false, + }; + } }; -} +}; -/// Creates a new `Serializer` with the given writer and options. -pub fn serializer(writer: anytype, options: SerializerOptions) Serializer(@TypeOf(writer)) { - return .init(writer, options); +test Serializer { + var discarding: Writer.Discarding = .init(&.{}); + var s: Serializer = .{ .writer = &discarding.writer }; + var vec2 = try s.beginStruct(.{}); + try vec2.field("x", 1.5, .{}); + try vec2.fieldPrefix("prefix"); + try s.value(2.5, .{}); + try vec2.end(); } fn expectSerializeEqual( @@ -1075,10 +1051,12 @@ fn expectSerializeEqual( value: anytype, options: SerializeOptions, ) !void { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - try serialize(value, options, buf.writer()); - try std.testing.expectEqualStrings(expected, buf.items); + var aw: Writer.Allocating = .init(std.testing.allocator); + const bw = &aw.writer; + defer aw.deinit(); + + try serialize(value, options, bw); + try std.testing.expectEqualStrings(expected, aw.getWritten()); } test "std.zon stringify whitespace, high level API" { @@ -1175,59 +1153,59 @@ test "std.zon stringify whitespace, high level API" { } test "std.zon stringify whitespace, low level API" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); - inline for (.{ true, false }) |whitespace| { - sz.options = .{ .whitespace = whitespace }; + for ([2]bool{ true, false }) |whitespace| { + s.options = .{ .whitespace = whitespace }; // Empty containers { - var container = try sz.beginStruct(.{}); + var container = try s.beginStruct(.{}); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{}); + var container = try s.beginTuple(.{}); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .fields = 0 } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .fields = 0 } }); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .fields = 0 } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .fields = 0 } }); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } // Size 1 { - var container = try sz.beginStruct(.{}); + var container = try s.beginStruct(.{}); try container.field("a", 1, .{}); try container.end(); if (whitespace) { @@ -1235,15 +1213,15 @@ test "std.zon stringify whitespace, low level API" { \\.{ \\ .a = 1, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1}", buf.items); + try std.testing.expectEqualStrings(".{.a=1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{}); + var container = try s.beginTuple(.{}); try container.field(1, .{}); try container.end(); if (whitespace) { @@ -1251,62 +1229,62 @@ test "std.zon stringify whitespace, low level API" { \\.{ \\ 1, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1}", buf.items); + try std.testing.expectEqualStrings(".{1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.field("a", 1, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1}", buf.items); + try std.testing.expectEqualStrings(".{.a=1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { // We get extra spaces here, since we didn't know up front that there would only be one // field. - var container = try sz.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); try container.field(1, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ 1 }", buf.items); + try std.testing.expectEqualStrings(".{ 1 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1}", buf.items); + try std.testing.expectEqualStrings(".{1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); try container.field("a", 1, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1}", buf.items); + try std.testing.expectEqualStrings(".{.a=1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .fields = 1 } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .fields = 1 } }); try container.field(1, .{}); try container.end(); - try std.testing.expectEqualStrings(".{1}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{1}", aw.getWritten()); + aw.clearRetainingCapacity(); } // Size 2 { - var container = try sz.beginStruct(.{}); + var container = try s.beginStruct(.{}); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.end(); @@ -1316,15 +1294,15 @@ test "std.zon stringify whitespace, low level API" { \\ .a = 1, \\ .b = 2, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{}); + var container = try s.beginTuple(.{}); try container.field(1, .{}); try container.field(2, .{}); try container.end(); @@ -1334,68 +1312,68 @@ test "std.zon stringify whitespace, low level API" { \\ 1, \\ 2, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2}", buf.items); + try std.testing.expectEqualStrings(".{1,2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1, .b = 2 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1, .b = 2 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); try container.field(1, .{}); try container.field(2, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2}", buf.items); + try std.testing.expectEqualStrings(".{1,2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .fields = 2 } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .fields = 2 } }); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1, .b = 2 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1, .b = 2 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .fields = 2 } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .fields = 2 } }); try container.field(1, .{}); try container.field(2, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2}", buf.items); + try std.testing.expectEqualStrings(".{1,2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } // Size 3 { - var container = try sz.beginStruct(.{}); + var container = try s.beginStruct(.{}); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.field("c", 3, .{}); @@ -1407,15 +1385,15 @@ test "std.zon stringify whitespace, low level API" { \\ .b = 2, \\ .c = 3, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{}); + var container = try s.beginTuple(.{}); try container.field(1, .{}); try container.field(2, .{}); try container.field(3, .{}); @@ -1427,43 +1405,43 @@ test "std.zon stringify whitespace, low level API" { \\ 2, \\ 3, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2,3}", buf.items); + try std.testing.expectEqualStrings(".{1,2,3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.field("c", 3, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1, .b = 2, .c = 3 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1, .b = 2, .c = 3 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); try container.field(1, .{}); try container.field(2, .{}); try container.field(3, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ 1, 2, 3 }", buf.items); + try std.testing.expectEqualStrings(".{ 1, 2, 3 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2,3}", buf.items); + try std.testing.expectEqualStrings(".{1,2,3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .fields = 3 } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .fields = 3 } }); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.field("c", 3, .{}); @@ -1475,15 +1453,15 @@ test "std.zon stringify whitespace, low level API" { \\ .b = 2, \\ .c = 3, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .fields = 3 } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .fields = 3 } }); try container.field(1, .{}); try container.field(2, .{}); try container.field(3, .{}); @@ -1495,16 +1473,16 @@ test "std.zon stringify whitespace, low level API" { \\ 2, \\ 3, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2,3}", buf.items); + try std.testing.expectEqualStrings(".{1,2,3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } // Nested objects where the outer container doesn't wrap but the inner containers do { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.field("first", .{ 1, 2, 3 }, .{}); try container.field("second", .{ 4, 5, 6 }, .{}); try container.end(); @@ -1519,139 +1497,141 @@ test "std.zon stringify whitespace, low level API" { \\ 5, \\ 6, \\} } - , buf.items); + , aw.getWritten()); } else { try std.testing.expectEqualStrings( ".{.first=.{1,2,3},.second=.{4,5,6}}", - buf.items, + aw.getWritten(), ); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } } } test "std.zon stringify utf8 codepoints" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Printable ASCII - try sz.int('a'); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.int('a'); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.codePoint('a'); - try std.testing.expectEqualStrings("'a'", buf.items); - buf.clearRetainingCapacity(); + try s.codePoint('a'); + try std.testing.expectEqualStrings("'a'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('a', .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("'a'", buf.items); - buf.clearRetainingCapacity(); + try s.value('a', .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("'a'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('a', .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("'a'", buf.items); - buf.clearRetainingCapacity(); + try s.value('a', .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("'a'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('a', .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.value('a', .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); // Short escaped codepoint - try sz.int('\n'); - try std.testing.expectEqualStrings("10", buf.items); - buf.clearRetainingCapacity(); + try s.int('\n'); + try std.testing.expectEqualStrings("10", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.codePoint('\n'); - try std.testing.expectEqualStrings("'\\n'", buf.items); - buf.clearRetainingCapacity(); + try s.codePoint('\n'); + try std.testing.expectEqualStrings("'\\n'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('\n', .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("'\\n'", buf.items); - buf.clearRetainingCapacity(); + try s.value('\n', .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("'\\n'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('\n', .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("10", buf.items); - buf.clearRetainingCapacity(); + try s.value('\n', .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("10", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('\n', .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("10", buf.items); - buf.clearRetainingCapacity(); + try s.value('\n', .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("10", aw.getWritten()); + aw.clearRetainingCapacity(); // Large codepoint - try sz.int('âš¡'); - try std.testing.expectEqualStrings("9889", buf.items); - buf.clearRetainingCapacity(); + try s.int('âš¡'); + try std.testing.expectEqualStrings("9889", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.codePoint('âš¡'); - try std.testing.expectEqualStrings("'\\xe2\\x9a\\xa1'", buf.items); - buf.clearRetainingCapacity(); + try s.codePoint('âš¡'); + try std.testing.expectEqualStrings("'\\u{26a1}'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('âš¡', .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("'\\xe2\\x9a\\xa1'", buf.items); - buf.clearRetainingCapacity(); + try s.value('âš¡', .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("'\\u{26a1}'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('âš¡', .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("9889", buf.items); - buf.clearRetainingCapacity(); + try s.value('âš¡', .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("9889", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('âš¡', .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("9889", buf.items); - buf.clearRetainingCapacity(); + try s.value('âš¡', .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("9889", aw.getWritten()); + aw.clearRetainingCapacity(); // Invalid codepoint - try std.testing.expectError(error.InvalidCodepoint, sz.codePoint(0x110000 + 1)); + try s.codePoint(0x110000 + 1); + try std.testing.expectEqualStrings("'\\u{110001}'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.int(0x110000 + 1); - try std.testing.expectEqualStrings("1114113", buf.items); - buf.clearRetainingCapacity(); + try s.int(0x110000 + 1); + try std.testing.expectEqualStrings("1114113", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(0x110000 + 1, .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("1114113", buf.items); - buf.clearRetainingCapacity(); + try s.value(0x110000 + 1, .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("1114113", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(0x110000 + 1, .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("1114113", buf.items); - buf.clearRetainingCapacity(); + try s.value(0x110000 + 1, .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("1114113", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(0x110000 + 1, .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("1114113", buf.items); - buf.clearRetainingCapacity(); + try s.value(0x110000 + 1, .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("1114113", aw.getWritten()); + aw.clearRetainingCapacity(); // Valid codepoint, not a codepoint type - try sz.value(@as(u22, 'a'), .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.value(@as(u22, 'a'), .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(@as(u22, 'a'), .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.value(@as(u22, 'a'), .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(@as(i32, 'a'), .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.value(@as(i32, 'a'), .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); // Make sure value options are passed to children - try sz.value(.{ .c = 'âš¡' }, .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings(".{ .c = '\\xe2\\x9a\\xa1' }", buf.items); - buf.clearRetainingCapacity(); + try s.value(.{ .c = 'âš¡' }, .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings(".{ .c = '\\u{26a1}' }", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(.{ .c = 'âš¡' }, .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings(".{ .c = 9889 }", buf.items); - buf.clearRetainingCapacity(); + try s.value(.{ .c = 'âš¡' }, .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings(".{ .c = 9889 }", aw.getWritten()); + aw.clearRetainingCapacity(); } test "std.zon stringify strings" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Minimal case - try sz.string("abcâš¡\n"); - try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", buf.items); - buf.clearRetainingCapacity(); + try s.string("abcâš¡\n"); + try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.tuple("abcâš¡\n", .{}); + try s.tuple("abcâš¡\n", .{}); try std.testing.expectEqualStrings( \\.{ \\ 97, @@ -1662,14 +1642,14 @@ test "std.zon stringify strings" { \\ 161, \\ 10, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value("abcâš¡\n", .{}); - try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", buf.items); - buf.clearRetainingCapacity(); + try s.value("abcâš¡\n", .{}); + try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value("abcâš¡\n", .{ .emit_strings_as_containers = true }); + try s.value("abcâš¡\n", .{ .emit_strings_as_containers = true }); try std.testing.expectEqualStrings( \\.{ \\ 97, @@ -1680,113 +1660,113 @@ test "std.zon stringify strings" { \\ 161, \\ 10, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); // Value options are inherited by children - try sz.value(.{ .str = "abc" }, .{}); - try std.testing.expectEqualStrings(".{ .str = \"abc\" }", buf.items); - buf.clearRetainingCapacity(); + try s.value(.{ .str = "abc" }, .{}); + try std.testing.expectEqualStrings(".{ .str = \"abc\" }", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(.{ .str = "abc" }, .{ .emit_strings_as_containers = true }); + try s.value(.{ .str = "abc" }, .{ .emit_strings_as_containers = true }); try std.testing.expectEqualStrings( \\.{ .str = .{ \\ 97, \\ 98, \\ 99, \\} } - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); // Arrays (rather than pointers to arrays) of u8s are not considered strings, so that data can // round trip correctly. - try sz.value("abc".*, .{}); + try s.value("abc".*, .{}); try std.testing.expectEqualStrings( \\.{ \\ 97, \\ 98, \\ 99, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); } test "std.zon stringify multiline strings" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); inline for (.{ true, false }) |whitespace| { - sz.options.whitespace = whitespace; + s.options.whitespace = whitespace; { - try sz.multilineString("", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("abcâš¡", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\abcâš¡", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("abcâš¡", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\abcâš¡", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("abcâš¡\ndef", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\abcâš¡\n\\\\def", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("abcâš¡\ndef", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\abcâš¡\n\\\\def", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("abcâš¡\r\ndef", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\abcâš¡\n\\\\def", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("abcâš¡\r\ndef", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\abcâš¡\n\\\\def", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("\nabcâš¡", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\\n\\\\abcâš¡", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("\nabcâš¡", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\\n\\\\abcâš¡", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("\r\nabcâš¡", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\\n\\\\abcâš¡", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("\r\nabcâš¡", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\\n\\\\abcâš¡", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("abc\ndef", .{}); + try s.multilineString("abc\ndef", .{}); if (whitespace) { - try std.testing.expectEqualStrings("\n\\\\abc\n\\\\def\n", buf.items); + try std.testing.expectEqualStrings("\n\\\\abc\n\\\\def\n", aw.getWritten()); } else { - try std.testing.expectEqualStrings("\\\\abc\n\\\\def\n", buf.items); + try std.testing.expectEqualStrings("\\\\abc\n\\\\def\n", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { const str: []const u8 = &.{ 'a', '\r', 'c' }; - try sz.string(str); - try std.testing.expectEqualStrings("\"a\\rc\"", buf.items); - buf.clearRetainingCapacity(); + try s.string(str); + try std.testing.expectEqualStrings("\"a\\rc\"", aw.getWritten()); + aw.clearRetainingCapacity(); } { try std.testing.expectError( error.InnerCarriageReturn, - sz.multilineString(@as([]const u8, &.{ 'a', '\r', 'c' }), .{}), + s.multilineString(@as([]const u8, &.{ 'a', '\r', 'c' }), .{}), ); try std.testing.expectError( error.InnerCarriageReturn, - sz.multilineString(@as([]const u8, &.{ 'a', '\r', 'c', '\n' }), .{}), + s.multilineString(@as([]const u8, &.{ 'a', '\r', 'c', '\n' }), .{}), ); try std.testing.expectError( error.InnerCarriageReturn, - sz.multilineString(@as([]const u8, &.{ 'a', '\r', 'c', '\r', '\n' }), .{}), + s.multilineString(@as([]const u8, &.{ 'a', '\r', 'c', '\r', '\n' }), .{}), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); } } } @@ -1932,42 +1912,43 @@ test "std.zon stringify skip default fields" { } test "std.zon depth limits" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); + var aw: Writer.Allocating = .init(std.testing.allocator); + const bw = &aw.writer; + defer aw.deinit(); const Recurse = struct { r: []const @This() }; // Normal operation - try serializeMaxDepth(.{ 1, .{ 2, 3 } }, .{}, buf.writer(), 16); - try std.testing.expectEqualStrings(".{ 1, .{ 2, 3 } }", buf.items); - buf.clearRetainingCapacity(); + try serializeMaxDepth(.{ 1, .{ 2, 3 } }, .{}, bw, 16); + try std.testing.expectEqualStrings(".{ 1, .{ 2, 3 } }", aw.getWritten()); + aw.clearRetainingCapacity(); - try serializeArbitraryDepth(.{ 1, .{ 2, 3 } }, .{}, buf.writer()); - try std.testing.expectEqualStrings(".{ 1, .{ 2, 3 } }", buf.items); - buf.clearRetainingCapacity(); + try serializeArbitraryDepth(.{ 1, .{ 2, 3 } }, .{}, bw); + try std.testing.expectEqualStrings(".{ 1, .{ 2, 3 } }", aw.getWritten()); + aw.clearRetainingCapacity(); // Max depth failing on non recursive type try std.testing.expectError( error.ExceededMaxDepth, - serializeMaxDepth(.{ 1, .{ 2, .{ 3, 4 } } }, .{}, buf.writer(), 3), + serializeMaxDepth(.{ 1, .{ 2, .{ 3, 4 } } }, .{}, bw, 3), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); // Max depth passing on recursive type { const maybe_recurse = Recurse{ .r = &.{} }; - try serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 2); - try std.testing.expectEqualStrings(".{ .r = .{} }", buf.items); - buf.clearRetainingCapacity(); + try serializeMaxDepth(maybe_recurse, .{}, bw, 2); + try std.testing.expectEqualStrings(".{ .r = .{} }", aw.getWritten()); + aw.clearRetainingCapacity(); } // Unchecked passing on recursive type { const maybe_recurse = Recurse{ .r = &.{} }; - try serializeArbitraryDepth(maybe_recurse, .{}, buf.writer()); - try std.testing.expectEqualStrings(".{ .r = .{} }", buf.items); - buf.clearRetainingCapacity(); + try serializeArbitraryDepth(maybe_recurse, .{}, bw); + try std.testing.expectEqualStrings(".{ .r = .{} }", aw.getWritten()); + aw.clearRetainingCapacity(); } // Max depth failing on recursive type due to depth @@ -1976,10 +1957,10 @@ test "std.zon depth limits" { maybe_recurse.r = &.{.{ .r = &.{} }}; try std.testing.expectError( error.ExceededMaxDepth, - serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 2), + serializeMaxDepth(maybe_recurse, .{}, bw, 2), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); } // Same but for a slice @@ -1989,23 +1970,23 @@ test "std.zon depth limits" { try std.testing.expectError( error.ExceededMaxDepth, - serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 2), + serializeMaxDepth(maybe_recurse, .{}, bw, 2), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); - var sz = serializer(buf.writer(), .{}); + var s: Serializer = .{ .writer = bw }; try std.testing.expectError( error.ExceededMaxDepth, - sz.tupleMaxDepth(maybe_recurse, .{}, 2), + s.tupleMaxDepth(maybe_recurse, .{}, 2), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.tupleArbitraryDepth(maybe_recurse, .{}); - try std.testing.expectEqualStrings(".{.{ .r = .{} }}", buf.items); - buf.clearRetainingCapacity(); + try s.tupleArbitraryDepth(maybe_recurse, .{}); + try std.testing.expectEqualStrings(".{.{ .r = .{} }}", aw.getWritten()); + aw.clearRetainingCapacity(); } // A slice succeeding @@ -2013,19 +1994,19 @@ test "std.zon depth limits" { var temp: [1]Recurse = .{.{ .r = &.{} }}; const maybe_recurse: []const Recurse = &temp; - try serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 3); - try std.testing.expectEqualStrings(".{.{ .r = .{} }}", buf.items); - buf.clearRetainingCapacity(); + try serializeMaxDepth(maybe_recurse, .{}, bw, 3); + try std.testing.expectEqualStrings(".{.{ .r = .{} }}", aw.getWritten()); + aw.clearRetainingCapacity(); - var sz = serializer(buf.writer(), .{}); + var s: Serializer = .{ .writer = bw }; - try sz.tupleMaxDepth(maybe_recurse, .{}, 3); - try std.testing.expectEqualStrings(".{.{ .r = .{} }}", buf.items); - buf.clearRetainingCapacity(); + try s.tupleMaxDepth(maybe_recurse, .{}, 3); + try std.testing.expectEqualStrings(".{.{ .r = .{} }}", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.tupleArbitraryDepth(maybe_recurse, .{}); - try std.testing.expectEqualStrings(".{.{ .r = .{} }}", buf.items); - buf.clearRetainingCapacity(); + try s.tupleArbitraryDepth(maybe_recurse, .{}); + try std.testing.expectEqualStrings(".{.{ .r = .{} }}", aw.getWritten()); + aw.clearRetainingCapacity(); } // Max depth failing on recursive type due to recursion @@ -2036,46 +2017,46 @@ test "std.zon depth limits" { try std.testing.expectError( error.ExceededMaxDepth, - serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 128), + serializeMaxDepth(maybe_recurse, .{}, bw, 128), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); - var sz = serializer(buf.writer(), .{}); + var s: Serializer = .{ .writer = bw }; try std.testing.expectError( error.ExceededMaxDepth, - sz.tupleMaxDepth(maybe_recurse, .{}, 128), + s.tupleMaxDepth(maybe_recurse, .{}, 128), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); } // Max depth on other parts of the lower level API { - var sz = serializer(buf.writer(), .{}); + var s: Serializer = .{ .writer = bw }; const maybe_recurse: []const Recurse = &.{}; - try std.testing.expectError(error.ExceededMaxDepth, sz.valueMaxDepth(1, .{}, 0)); - try sz.valueMaxDepth(2, .{}, 1); - try sz.value(3, .{}); - try sz.valueArbitraryDepth(maybe_recurse, .{}); + try std.testing.expectError(error.ExceededMaxDepth, s.valueMaxDepth(1, .{}, 0)); + try s.valueMaxDepth(2, .{}, 1); + try s.value(3, .{}); + try s.valueArbitraryDepth(maybe_recurse, .{}); - var s = try sz.beginStruct(.{}); - try std.testing.expectError(error.ExceededMaxDepth, s.fieldMaxDepth("a", 1, .{}, 0)); - try s.fieldMaxDepth("b", 4, .{}, 1); - try s.field("c", 5, .{}); - try s.fieldArbitraryDepth("d", maybe_recurse, .{}); - try s.end(); + var wip_struct = try s.beginStruct(.{}); + try std.testing.expectError(error.ExceededMaxDepth, wip_struct.fieldMaxDepth("a", 1, .{}, 0)); + try wip_struct.fieldMaxDepth("b", 4, .{}, 1); + try wip_struct.field("c", 5, .{}); + try wip_struct.fieldArbitraryDepth("d", maybe_recurse, .{}); + try wip_struct.end(); - var t = try sz.beginTuple(.{}); + var t = try s.beginTuple(.{}); try std.testing.expectError(error.ExceededMaxDepth, t.fieldMaxDepth(1, .{}, 0)); try t.fieldMaxDepth(6, .{}, 1); try t.field(7, .{}); try t.fieldArbitraryDepth(maybe_recurse, .{}); try t.end(); - var a = try sz.beginTuple(.{}); + var a = try s.beginTuple(.{}); try std.testing.expectError(error.ExceededMaxDepth, a.fieldMaxDepth(1, .{}, 0)); try a.fieldMaxDepth(8, .{}, 1); try a.field(9, .{}); @@ -2096,7 +2077,7 @@ test "std.zon depth limits" { \\ 9, \\ .{}, \\} - , buf.items); + , aw.getWritten()); } } @@ -2192,42 +2173,42 @@ test "std.zon stringify primitives" { } test "std.zon stringify ident" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); try expectSerializeEqual(".{ .a = 0 }", .{ .a = 0 }, .{}); - try sz.ident("a"); - try std.testing.expectEqualStrings(".a", buf.items); - buf.clearRetainingCapacity(); + try s.ident("a"); + try std.testing.expectEqualStrings(".a", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("foo_1"); - try std.testing.expectEqualStrings(".foo_1", buf.items); - buf.clearRetainingCapacity(); + try s.ident("foo_1"); + try std.testing.expectEqualStrings(".foo_1", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("_foo_1"); - try std.testing.expectEqualStrings("._foo_1", buf.items); - buf.clearRetainingCapacity(); + try s.ident("_foo_1"); + try std.testing.expectEqualStrings("._foo_1", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("foo bar"); - try std.testing.expectEqualStrings(".@\"foo bar\"", buf.items); - buf.clearRetainingCapacity(); + try s.ident("foo bar"); + try std.testing.expectEqualStrings(".@\"foo bar\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("1foo"); - try std.testing.expectEqualStrings(".@\"1foo\"", buf.items); - buf.clearRetainingCapacity(); + try s.ident("1foo"); + try std.testing.expectEqualStrings(".@\"1foo\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("var"); - try std.testing.expectEqualStrings(".@\"var\"", buf.items); - buf.clearRetainingCapacity(); + try s.ident("var"); + try std.testing.expectEqualStrings(".@\"var\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("true"); - try std.testing.expectEqualStrings(".true", buf.items); - buf.clearRetainingCapacity(); + try s.ident("true"); + try std.testing.expectEqualStrings(".true", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("_"); - try std.testing.expectEqualStrings("._", buf.items); - buf.clearRetainingCapacity(); + try s.ident("_"); + try std.testing.expectEqualStrings("._", aw.getWritten()); + aw.clearRetainingCapacity(); const Enum = enum { @"foo bar", @@ -2239,40 +2220,40 @@ test "std.zon stringify ident" { } test "std.zon stringify as tuple" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Tuples - try sz.tuple(.{ 1, 2 }, .{}); - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); - buf.clearRetainingCapacity(); + try s.tuple(.{ 1, 2 }, .{}); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); + aw.clearRetainingCapacity(); // Slice - try sz.tuple(@as([]const u8, &.{ 1, 2 }), .{}); - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); - buf.clearRetainingCapacity(); + try s.tuple(@as([]const u8, &.{ 1, 2 }), .{}); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); + aw.clearRetainingCapacity(); // Array - try sz.tuple([2]u8{ 1, 2 }, .{}); - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); - buf.clearRetainingCapacity(); + try s.tuple([2]u8{ 1, 2 }, .{}); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); + aw.clearRetainingCapacity(); } test "std.zon stringify as float" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Comptime float - try sz.float(2.5); - try std.testing.expectEqualStrings("2.5", buf.items); - buf.clearRetainingCapacity(); + try s.float(2.5); + try std.testing.expectEqualStrings("2.5", aw.getWritten()); + aw.clearRetainingCapacity(); // Sized float - try sz.float(@as(f32, 2.5)); - try std.testing.expectEqualStrings("2.5", buf.items); - buf.clearRetainingCapacity(); + try s.float(@as(f32, 2.5)); + try std.testing.expectEqualStrings("2.5", aw.getWritten()); + aw.clearRetainingCapacity(); } test "std.zon stringify vector" { @@ -2364,13 +2345,13 @@ test "std.zon pointers" { } test "std.zon tuple/struct field" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Test on structs { - var root = try sz.beginStruct(.{}); + var root = try s.beginStruct(.{}); { var tuple = try root.beginTupleField("foo", .{}); try tuple.field(0, .{}); @@ -2396,13 +2377,13 @@ test "std.zon tuple/struct field" { \\ .b = 1, \\ }, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); } // Test on tuples { - var root = try sz.beginTuple(.{}); + var root = try s.beginTuple(.{}); { var tuple = try root.beginTupleField(.{}); try tuple.field(0, .{}); @@ -2428,7 +2409,7 @@ test "std.zon tuple/struct field" { \\ .b = 1, \\ }, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); } } -- 2.54.0 From 737b13357ed04a33bfea17f58674deba8f17f51e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 17:11:47 -0700 Subject: [PATCH 08/26] resinator: fix std.json API usage --- lib/compiler/resinator/main.zig | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index 903e0a2f719a3beb20e80c07e7b08814a988954c..29cf716600dcbc63577084b7c547d04d8972c160 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -290,12 +290,14 @@ pub fn main() !void { }; defer depfile.close(); - const depfile_writer = depfile.deprecatedWriter(); - var depfile_buffered_writer = std.io.bufferedWriter(depfile_writer); + var depfile_buffer: [1024]u8 = undefined; + var depfile_writer = depfile.writer(&depfile_buffer); switch (options.depfile_fmt) { .json => { - var write_stream = std.json.writeStream(depfile_buffered_writer.writer(), .{ .whitespace = .indent_2 }); - defer write_stream.deinit(); + var write_stream: std.json.Stringify = .{ + .writer = &depfile_writer.interface, + .options = .{ .whitespace = .indent_2 }, + }; try write_stream.beginArray(); for (dependencies_list.items) |dep_path| { @@ -304,7 +306,7 @@ pub fn main() !void { try write_stream.endArray(); }, } - try depfile_buffered_writer.flush(); + try depfile_writer.interface.flush(); } } -- 2.54.0 From 0fb7a0a94bf6f9e329008d8b5b819a8d1d7124b0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 17:33:44 -0700 Subject: [PATCH 09/26] std.zon: better namespace for Serializer --- lib/std/zon.zig | 1 + lib/std/zon/Serializer.zig | 929 ++++++++++++++++++++++++++++++++++++ lib/std/zon/stringify.zig | 936 +------------------------------------ src/main.zig | 9 +- src/print_env.zig | 51 +- src/print_targets.zig | 12 +- src/translate_c.zig | 2 +- 7 files changed, 958 insertions(+), 982 deletions(-) create mode 100644 lib/std/zon/Serializer.zig diff --git a/lib/std/zon.zig b/lib/std/zon.zig index 252331057a8c16ad0c4acdcfeef33a6834e05a8e..9ac02cf74128a7f5d0e981e88ed8af40505b1eee 100644 --- a/lib/std/zon.zig +++ b/lib/std/zon.zig @@ -38,6 +38,7 @@ pub const parse = @import("zon/parse.zig"); pub const stringify = @import("zon/stringify.zig"); +pub const Serializer = @import("zon/Serializer.zig"); test { _ = parse; diff --git a/lib/std/zon/Serializer.zig b/lib/std/zon/Serializer.zig new file mode 100644 index 0000000000000000000000000000000000000000..b65b13bf971f6c842931b0ab097396eddd08bddb --- /dev/null +++ b/lib/std/zon/Serializer.zig @@ -0,0 +1,929 @@ +//! Lower level control over serialization, you can create a new instance with `serializer`. +//! +//! Useful when you want control over which fields are serialized, how they're represented, +//! or want to write a ZON object that does not exist in memory. +//! +//! You can serialize values with `value`. To serialize recursive types, the following are provided: +//! * `valueMaxDepth` +//! * `valueArbitraryDepth` +//! +//! You can also serialize values using specific notations: +//! * `int` +//! * `float` +//! * `codePoint` +//! * `tuple` +//! * `tupleMaxDepth` +//! * `tupleArbitraryDepth` +//! * `string` +//! * `multilineString` +//! +//! For manual serialization of containers, see: +//! * `beginStruct` +//! * `beginTuple` + +options: Options = .{}, +indent_level: u8 = 0, +writer: *Writer, + +const Serializer = @This(); +const std = @import("std"); +const assert = std.debug.assert; +const Writer = std.Io.Writer; + +pub const Error = Writer.Error; +pub const DepthError = Error || error{ExceededMaxDepth}; + +pub const Options = struct { + /// If false, only syntactically necessary whitespace is emitted. + whitespace: bool = true, +}; + +/// Options for manual serialization of container types. +pub const ContainerOptions = struct { + /// The whitespace style that should be used for this container. Ignored if whitespace is off. + whitespace_style: union(enum) { + /// If true, wrap every field. If false do not. + wrap: bool, + /// Automatically decide whether to wrap or not based on the number of fields. Following + /// the standard rule of thumb, containers with more than two fields are wrapped. + fields: usize, + } = .{ .wrap = true }, + + fn shouldWrap(self: ContainerOptions) bool { + return switch (self.whitespace_style) { + .wrap => |wrap| wrap, + .fields => |fields| fields > 2, + }; + } +}; + +/// Options for serialization of an individual value. +/// +/// See `SerializeOptions` for more information on these options. +pub const ValueOptions = struct { + emit_codepoint_literals: EmitCodepointLiterals = .never, + emit_strings_as_containers: bool = false, + emit_default_optional_fields: bool = true, +}; + +/// Determines when to emit Unicode code point literals as opposed to integer literals. +pub const EmitCodepointLiterals = enum { + /// Never emit Unicode code point literals. + never, + /// Emit Unicode code point literals for any `u8` in the printable ASCII range. + printable_ascii, + /// Emit Unicode code point literals for any unsigned integer with 21 bits or fewer + /// whose value is a valid non-surrogate code point. + always, + + /// If the value should be emitted as a Unicode codepoint, return it as a u21. + fn emitAsCodepoint(self: @This(), val: anytype) ?u21 { + // Rule out incompatible integer types + switch (@typeInfo(@TypeOf(val))) { + .int => |int_info| if (int_info.signedness == .signed or int_info.bits > 21) { + return null; + }, + .comptime_int => {}, + else => comptime unreachable, + } + + // Return null if the value shouldn't be printed as a Unicode codepoint, or the value casted + // to a u21 if it should. + switch (self) { + .always => { + const c = std.math.cast(u21, val) orelse return null; + if (!std.unicode.utf8ValidCodepoint(c)) return null; + return c; + }, + .printable_ascii => { + const c = std.math.cast(u8, val) orelse return null; + if (!std.ascii.isPrint(c)) return null; + return c; + }, + .never => { + return null; + }, + } + } +}; + +/// Serialize a value, similar to `serialize`. +pub fn value(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + return self.valueArbitraryDepth(val, options); +} + +/// Serialize a value, similar to `serializeMaxDepth`. +/// Can return `error.ExceededMaxDepth`. +pub fn valueMaxDepth(self: *Serializer, val: anytype, options: ValueOptions, depth: usize) DepthError!void { + try checkValueDepth(val, depth); + return self.valueArbitraryDepth(val, options); +} + +/// Serialize a value, similar to `serializeArbitraryDepth`. +pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(canSerializeType(@TypeOf(val))); + switch (@typeInfo(@TypeOf(val))) { + .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { + self.codePoint(c) catch |err| switch (err) { + error.InvalidCodepoint => unreachable, // Already validated + else => |e| return e, + }; + } else { + try self.int(val); + }, + .float, .comptime_float => try self.float(val), + .bool, .null => try self.writer.print("{}", .{val}), + .enum_literal => try self.ident(@tagName(val)), + .@"enum" => try self.ident(@tagName(val)), + .pointer => |pointer| { + // Try to serialize as a string + const item: ?type = switch (@typeInfo(pointer.child)) { + .array => |array| array.child, + else => if (pointer.size == .slice) pointer.child else null, + }; + if (item == u8 and + (pointer.sentinel() == null or pointer.sentinel() == 0) and + !options.emit_strings_as_containers) + { + return try self.string(val); + } + + // Serialize as either a tuple or as the child type + switch (pointer.size) { + .slice => try self.tupleImpl(val, options), + .one => try self.valueArbitraryDepth(val.*, options), + else => comptime unreachable, + } + }, + .array => { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = val.len } }, + ); + for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + .@"struct" => |@"struct"| if (@"struct".is_tuple) { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = @"struct".fields.len } }, + ); + inline for (val) |field_value| { + try container.fieldArbitraryDepth(field_value, options); + } + try container.end(); + } else { + // Decide which fields to emit + const fields, const skipped: [@"struct".fields.len]bool = if (options.emit_default_optional_fields) b: { + break :b .{ @"struct".fields.len, @splat(false) }; + } else b: { + var fields = @"struct".fields.len; + var skipped: [@"struct".fields.len]bool = @splat(false); + inline for (@"struct".fields, &skipped) |field_info, *skip| { + if (field_info.default_value_ptr) |ptr| { + const default: *const field_info.type = @ptrCast(@alignCast(ptr)); + const field_value = @field(val, field_info.name); + if (std.meta.eql(field_value, default.*)) { + skip.* = true; + fields -= 1; + } + } + } + break :b .{ fields, skipped }; + }; + + // Emit those fields + var container = try self.beginStruct( + .{ .whitespace_style = .{ .fields = fields } }, + ); + inline for (@"struct".fields, skipped) |field_info, skip| { + if (!skip) { + try container.fieldArbitraryDepth( + field_info.name, + @field(val, field_info.name), + options, + ); + } + } + try container.end(); + }, + .@"union" => |@"union"| { + comptime assert(@"union".tag_type != null); + switch (val) { + inline else => |pl, tag| if (@TypeOf(pl) == void) + try self.writer.print(".{s}", .{@tagName(tag)}) + else { + var container = try self.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); + + try container.fieldArbitraryDepth( + @tagName(tag), + pl, + options, + ); + + try container.end(); + }, + } + }, + .optional => if (val) |inner| { + try self.valueArbitraryDepth(inner, options); + } else { + try self.writer.writeAll("null"); + }, + .vector => |vector| { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = vector.len } }, + ); + for (0..vector.len) |i| { + try container.fieldArbitraryDepth(val[i], options); + } + try container.end(); + }, + + else => comptime unreachable, + } +} + +/// Serialize an integer. +pub fn int(self: *Serializer, val: anytype) Error!void { + try self.writer.printInt(val, 10, .lower, .{}); +} + +/// Serialize a float. +pub fn float(self: *Serializer, val: anytype) Error!void { + switch (@typeInfo(@TypeOf(val))) { + .float => if (std.math.isNan(val)) { + return self.writer.writeAll("nan"); + } else if (std.math.isPositiveInf(val)) { + return self.writer.writeAll("inf"); + } else if (std.math.isNegativeInf(val)) { + return self.writer.writeAll("-inf"); + } else if (std.math.isNegativeZero(val)) { + return self.writer.writeAll("-0.0"); + } else { + try self.writer.print("{d}", .{val}); + }, + .comptime_float => if (val == 0) { + return self.writer.writeAll("0"); + } else { + try self.writer.print("{d}", .{val}); + }, + else => comptime unreachable, + } +} + +/// Serialize `name` as an identifier prefixed with `.`. +/// +/// Escapes the identifier if necessary. +pub fn ident(self: *Serializer, name: []const u8) Error!void { + try self.writer.print(".{f}", .{std.zig.fmtIdPU(name)}); +} + +pub const CodePointError = Error || error{InvalidCodepoint}; + +/// Serialize `val` as a Unicode codepoint. +/// +/// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint. +pub fn codePoint(self: *Serializer, val: u21) CodePointError!void { + try self.writer.print("'{f}'", .{std.zig.fmtChar(val)}); +} + +/// Like `value`, but always serializes `val` as a tuple. +/// +/// Will fail at comptime if `val` is not a tuple, array, pointer to an array, or slice. +pub fn tuple(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + try self.tupleArbitraryDepth(val, options); +} + +/// Like `tuple`, but recursive types are allowed. +/// +/// Returns `error.ExceededMaxDepth` if `depth` is exceeded. +pub fn tupleMaxDepth( + self: *Serializer, + val: anytype, + options: ValueOptions, + depth: usize, +) DepthError!void { + try checkValueDepth(val, depth); + try self.tupleArbitraryDepth(val, options); +} + +/// Like `tuple`, but recursive types are allowed. +/// +/// It is the caller's responsibility to ensure that `val` does not contain cycles. +pub fn tupleArbitraryDepth( + self: *Serializer, + val: anytype, + options: ValueOptions, +) Error!void { + try self.tupleImpl(val, options); +} + +fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(canSerializeType(@TypeOf(val))); + switch (@typeInfo(@TypeOf(val))) { + .@"struct" => { + var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); + inline for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + .pointer, .array => { + var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); + for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + else => comptime unreachable, + } +} + +/// Like `value`, but always serializes `val` as a string. +pub fn string(self: *Serializer, val: []const u8) Error!void { + try self.writer.print("\"{f}\"", .{std.zig.fmtString(val)}); +} + +/// Options for formatting multiline strings. +pub const MultilineStringOptions = struct { + /// If top level is true, whitespace before and after the multiline string is elided. + /// If it is true, a newline is printed, then the value, followed by a newline, and if + /// whitespace is true any necessary indentation follows. + top_level: bool = false, +}; + +pub const MultilineStringError = Error || error{InnerCarriageReturn}; + +/// Like `value`, but always serializes to a multiline string literal. +/// +/// Returns `error.InnerCarriageReturn` if `val` contains a CR not followed by a newline, +/// since multiline strings cannot represent CR without a following newline. +pub fn multilineString( + self: *Serializer, + val: []const u8, + options: MultilineStringOptions, +) MultilineStringError!void { + // Make sure the string does not contain any carriage returns not followed by a newline + var i: usize = 0; + while (i < val.len) : (i += 1) { + if (val[i] == '\r') { + if (i + 1 < val.len) { + if (val[i + 1] == '\n') { + i += 1; + continue; + } + } + return error.InnerCarriageReturn; + } + } + + if (!options.top_level) { + try self.newline(); + try self.indent(); + } + + try self.writer.writeAll("\\\\"); + for (val) |c| { + if (c != '\r') { + try self.writer.writeByte(c); // We write newlines here even if whitespace off + if (c == '\n') { + try self.indent(); + try self.writer.writeAll("\\\\"); + } + } + } + + if (!options.top_level) { + try self.writer.writeByte('\n'); // Even if whitespace off + try self.indent(); + } +} + +/// Create a `Struct` for writing ZON structs field by field. +pub fn beginStruct(self: *Serializer, options: ContainerOptions) Error!Struct { + return Struct.begin(self, options); +} + +/// Creates a `Tuple` for writing ZON tuples field by field. +pub fn beginTuple(self: *Serializer, options: ContainerOptions) Error!Tuple { + return Tuple.begin(self, options); +} + +fn indent(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.splatByteAll(' ', 4 * self.indent_level); + } +} + +fn newline(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.writeByte('\n'); + } +} + +fn newlineOrSpace(self: *Serializer, len: usize) Error!void { + if (self.containerShouldWrap(len)) { + try self.newline(); + } else { + try self.space(); + } +} + +fn space(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.writeByte(' '); + } +} + +/// Writes ZON tuples field by field. +pub const Tuple = struct { + container: Container, + + fn begin(parent: *Serializer, options: ContainerOptions) Error!Tuple { + return .{ + .container = try Container.begin(parent, .anon, options), + }; + } + + /// Finishes serializing the tuple. + /// + /// Prints a trailing comma as configured when appropriate, and the closing bracket. + pub fn end(self: *Tuple) Error!void { + try self.container.end(); + self.* = undefined; + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. + pub fn field( + self: *Tuple, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.field(null, val, options); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + pub fn fieldMaxDepth( + self: *Tuple, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try self.container.fieldMaxDepth(null, val, options, depth); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by + /// `valueArbitraryDepth`. + pub fn fieldArbitraryDepth( + self: *Tuple, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.fieldArbitraryDepth(null, val, options); + } + + /// Starts a field with a struct as a value. Returns the struct. + pub fn beginStructField( + self: *Tuple, + options: ContainerOptions, + ) Error!Struct { + try self.fieldPrefix(); + return self.container.serializer.beginStruct(options); + } + + /// Starts a field with a tuple as a value. Returns the tuple. + pub fn beginTupleField( + self: *Tuple, + options: ContainerOptions, + ) Error!Tuple { + try self.fieldPrefix(); + return self.container.serializer.beginTuple(options); + } + + /// Print a field prefix. This prints any necessary commas, and whitespace as + /// configured. Useful if you want to serialize the field value yourself. + pub fn fieldPrefix(self: *Tuple) Error!void { + try self.container.fieldPrefix(null); + } +}; + +/// Writes ZON structs field by field. +pub const Struct = struct { + container: Container, + + fn begin(parent: *Serializer, options: ContainerOptions) Error!Struct { + return .{ + .container = try Container.begin(parent, .named, options), + }; + } + + /// Finishes serializing the struct. + /// + /// Prints a trailing comma as configured when appropriate, and the closing bracket. + pub fn end(self: *Struct) Error!void { + try self.container.end(); + self.* = undefined; + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. + pub fn field( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.field(name, val, options); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + pub fn fieldMaxDepth( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try self.container.fieldMaxDepth(name, val, options, depth); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by + /// `valueArbitraryDepth`. + pub fn fieldArbitraryDepth( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.fieldArbitraryDepth(name, val, options); + } + + /// Starts a field with a struct as a value. Returns the struct. + pub fn beginStructField( + self: *Struct, + name: []const u8, + options: ContainerOptions, + ) Error!Struct { + try self.fieldPrefix(name); + return self.container.serializer.beginStruct(options); + } + + /// Starts a field with a tuple as a value. Returns the tuple. + pub fn beginTupleField( + self: *Struct, + name: []const u8, + options: ContainerOptions, + ) Error!Tuple { + try self.fieldPrefix(name); + return self.container.serializer.beginTuple(options); + } + + /// Print a field prefix. This prints any necessary commas, the field name (escaped if + /// necessary) and whitespace as configured. Useful if you want to serialize the field + /// value yourself. + pub fn fieldPrefix(self: *Struct, name: []const u8) Error!void { + try self.container.fieldPrefix(name); + } +}; + +const Container = struct { + const FieldStyle = enum { named, anon }; + + serializer: *Serializer, + field_style: FieldStyle, + options: ContainerOptions, + empty: bool, + + fn begin( + sz: *Serializer, + field_style: FieldStyle, + options: ContainerOptions, + ) Error!Container { + if (options.shouldWrap()) sz.indent_level +|= 1; + try sz.writer.writeAll(".{"); + return .{ + .serializer = sz, + .field_style = field_style, + .options = options, + .empty = true, + }; + } + + fn end(self: *Container) Error!void { + if (self.options.shouldWrap()) self.serializer.indent_level -|= 1; + if (!self.empty) { + if (self.options.shouldWrap()) { + if (self.serializer.options.whitespace) { + try self.serializer.writer.writeByte(','); + } + try self.serializer.newline(); + try self.serializer.indent(); + } else if (!self.shouldElideSpaces()) { + try self.serializer.space(); + } + } + try self.serializer.writer.writeByte('}'); + self.* = undefined; + } + + fn fieldPrefix(self: *Container, name: ?[]const u8) Error!void { + if (!self.empty) { + try self.serializer.writer.writeByte(','); + } + self.empty = false; + if (self.options.shouldWrap()) { + try self.serializer.newline(); + } else if (!self.shouldElideSpaces()) { + try self.serializer.space(); + } + if (self.options.shouldWrap()) try self.serializer.indent(); + if (name) |n| { + try self.serializer.ident(n); + try self.serializer.space(); + try self.serializer.writer.writeByte('='); + try self.serializer.space(); + } + } + + fn field( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + try self.fieldArbitraryDepth(name, val, options); + } + + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + fn fieldMaxDepth( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try checkValueDepth(val, depth); + try self.fieldArbitraryDepth(name, val, options); + } + + fn fieldArbitraryDepth( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.fieldPrefix(name); + try self.serializer.valueArbitraryDepth(val, options); + } + + fn shouldElideSpaces(self: *const Container) bool { + return switch (self.options.whitespace_style) { + .fields => |fields| self.field_style != .named and fields == 1, + else => false, + }; + } +}; + +test Serializer { + var discarding: Writer.Discarding = .init(&.{}); + var s: Serializer = .{ .writer = &discarding.writer }; + var vec2 = try s.beginStruct(.{}); + try vec2.field("x", 1.5, .{}); + try vec2.fieldPrefix("prefix"); + try s.value(2.5, .{}); + try vec2.end(); +} + +inline fn typeIsRecursive(comptime T: type) bool { + return comptime typeIsRecursiveInner(T, &.{}); +} + +fn typeIsRecursiveInner(comptime T: type, comptime prev_visited: []const type) bool { + for (prev_visited) |V| { + if (V == T) return true; + } + const visited = prev_visited ++ .{T}; + + return switch (@typeInfo(T)) { + .pointer => |pointer| typeIsRecursiveInner(pointer.child, visited), + .optional => |optional| typeIsRecursiveInner(optional.child, visited), + .array => |array| typeIsRecursiveInner(array.child, visited), + .vector => |vector| typeIsRecursiveInner(vector.child, visited), + .@"struct" => |@"struct"| for (@"struct".fields) |field| { + if (typeIsRecursiveInner(field.type, visited)) break true; + } else false, + .@"union" => |@"union"| inline for (@"union".fields) |field| { + if (typeIsRecursiveInner(field.type, visited)) break true; + } else false, + else => false, + }; +} + +test typeIsRecursive { + try std.testing.expect(!typeIsRecursive(bool)); + try std.testing.expect(!typeIsRecursive(struct { x: i32, y: i32 })); + try std.testing.expect(!typeIsRecursive(struct { i32, i32 })); + try std.testing.expect(typeIsRecursive(struct { x: i32, y: i32, z: *@This() })); + try std.testing.expect(typeIsRecursive(struct { + a: struct { + const A = @This(); + b: struct { + c: *struct { + a: ?A, + }, + }, + }, + })); + try std.testing.expect(typeIsRecursive(struct { + a: [3]*@This(), + })); + try std.testing.expect(typeIsRecursive(struct { + a: union { a: i32, b: *@This() }, + })); +} + +fn checkValueDepth(val: anytype, depth: usize) error{ExceededMaxDepth}!void { + if (depth == 0) return error.ExceededMaxDepth; + const child_depth = depth - 1; + + switch (@typeInfo(@TypeOf(val))) { + .pointer => |pointer| switch (pointer.size) { + .one => try checkValueDepth(val.*, child_depth), + .slice => for (val) |item| { + try checkValueDepth(item, child_depth); + }, + .c, .many => {}, + }, + .array => for (val) |item| { + try checkValueDepth(item, child_depth); + }, + .@"struct" => |@"struct"| inline for (@"struct".fields) |field_info| { + try checkValueDepth(@field(val, field_info.name), child_depth); + }, + .@"union" => |@"union"| if (@"union".tag_type == null) { + return; + } else switch (val) { + inline else => |payload| { + return checkValueDepth(payload, child_depth); + }, + }, + .optional => if (val) |inner| try checkValueDepth(inner, child_depth), + else => {}, + } +} + +fn expectValueDepthEquals(expected: usize, v: anytype) !void { + try checkValueDepth(v, expected); + try std.testing.expectError(error.ExceededMaxDepth, checkValueDepth(v, expected - 1)); +} + +test checkValueDepth { + try expectValueDepthEquals(1, 10); + try expectValueDepthEquals(2, .{ .x = 1, .y = 2 }); + try expectValueDepthEquals(2, .{ 1, 2 }); + try expectValueDepthEquals(3, .{ 1, .{ 2, 3 } }); + try expectValueDepthEquals(3, .{ .{ 1, 2 }, 3 }); + try expectValueDepthEquals(3, .{ .x = 0, .y = 1, .z = .{ .x = 3 } }); + try expectValueDepthEquals(3, .{ .x = 0, .y = .{ .x = 1 }, .z = 2 }); + try expectValueDepthEquals(3, .{ .x = .{ .x = 0 }, .y = 1, .z = 2 }); + try expectValueDepthEquals(2, @as(?u32, 1)); + try expectValueDepthEquals(1, @as(?u32, null)); + try expectValueDepthEquals(1, null); + try expectValueDepthEquals(2, &1); + try expectValueDepthEquals(3, &@as(?u32, 1)); + + const Union = union(enum) { + x: u32, + y: struct { x: u32 }, + }; + try expectValueDepthEquals(2, Union{ .x = 1 }); + try expectValueDepthEquals(3, Union{ .y = .{ .x = 1 } }); + + const Recurse = struct { r: ?*const @This() }; + try expectValueDepthEquals(2, Recurse{ .r = null }); + try expectValueDepthEquals(5, Recurse{ .r = &Recurse{ .r = null } }); + try expectValueDepthEquals(8, Recurse{ .r = &Recurse{ .r = &Recurse{ .r = null } } }); + + try expectValueDepthEquals(2, @as([]const u8, &.{ 1, 2, 3 })); + try expectValueDepthEquals(3, @as([]const []const u8, &.{&.{ 1, 2, 3 }})); +} + +inline fn canSerializeType(T: type) bool { + comptime return canSerializeTypeInner(T, &.{}, false); +} + +fn canSerializeTypeInner( + T: type, + /// Visited structs and unions, to avoid infinite recursion. + /// Tracking more types is unnecessary, and a little complex due to optional nesting. + visited: []const type, + parent_is_optional: bool, +) bool { + return switch (@typeInfo(T)) { + .bool, + .int, + .float, + .comptime_float, + .comptime_int, + .null, + .enum_literal, + => true, + + .noreturn, + .void, + .type, + .undefined, + .error_union, + .error_set, + .@"fn", + .frame, + .@"anyframe", + .@"opaque", + => false, + + .@"enum" => |@"enum"| @"enum".is_exhaustive, + + .pointer => |pointer| switch (pointer.size) { + .one => canSerializeTypeInner(pointer.child, visited, parent_is_optional), + .slice => canSerializeTypeInner(pointer.child, visited, false), + .many, .c => false, + }, + + .optional => |optional| if (parent_is_optional) + false + else + canSerializeTypeInner(optional.child, visited, true), + + .array => |array| canSerializeTypeInner(array.child, visited, false), + .vector => |vector| canSerializeTypeInner(vector.child, visited, false), + + .@"struct" => |@"struct"| { + for (visited) |V| if (T == V) return true; + const new_visited = visited ++ .{T}; + for (@"struct".fields) |field| { + if (!canSerializeTypeInner(field.type, new_visited, false)) return false; + } + return true; + }, + .@"union" => |@"union"| { + for (visited) |V| if (T == V) return true; + const new_visited = visited ++ .{T}; + if (@"union".tag_type == null) return false; + for (@"union".fields) |field| { + if (field.type != void and !canSerializeTypeInner(field.type, new_visited, false)) { + return false; + } + } + return true; + }, + }; +} + +test canSerializeType { + try std.testing.expect(!comptime canSerializeType(void)); + try std.testing.expect(!comptime canSerializeType(struct { f: [*]u8 })); + try std.testing.expect(!comptime canSerializeType(struct { error{foo} })); + try std.testing.expect(!comptime canSerializeType(union(enum) { a: void, f: [*c]u8 })); + try std.testing.expect(!comptime canSerializeType(@Vector(0, [*c]u8))); + try std.testing.expect(!comptime canSerializeType(*?[*c]u8)); + try std.testing.expect(!comptime canSerializeType(enum(u8) { _ })); + try std.testing.expect(!comptime canSerializeType(union { foo: void })); + try std.testing.expect(comptime canSerializeType(union(enum) { foo: void })); + try std.testing.expect(comptime canSerializeType(comptime_float)); + try std.testing.expect(comptime canSerializeType(comptime_int)); + try std.testing.expect(!comptime canSerializeType(struct { comptime foo: ??u8 = null })); + try std.testing.expect(comptime canSerializeType(@TypeOf(.foo))); + try std.testing.expect(comptime canSerializeType(?u8)); + try std.testing.expect(comptime canSerializeType(*?*u8)); + try std.testing.expect(comptime canSerializeType(?struct { + foo: ?struct { + ?union(enum) { + a: ?@Vector(0, ?*u8), + }, + ?struct { + f: ?[]?u8, + }, + }, + })); + try std.testing.expect(!comptime canSerializeType(??u8)); + try std.testing.expect(!comptime canSerializeType(?*?u8)); + try std.testing.expect(!comptime canSerializeType(*?*?*u8)); + try std.testing.expect(comptime canSerializeType(struct { x: comptime_int = 2 })); + try std.testing.expect(comptime canSerializeType(struct { x: comptime_float = 2 })); + try std.testing.expect(comptime canSerializeType(struct { comptime_int })); + try std.testing.expect(comptime canSerializeType(struct { comptime x: @TypeOf(.foo) = .foo })); + const Recursive = struct { foo: ?*@This() }; + try std.testing.expect(comptime canSerializeType(Recursive)); + + // Make sure we validate nested optional before we early out due to already having seen + // a type recursion! + try std.testing.expect(!comptime canSerializeType(struct { + add_to_visited: ?u8, + retrieve_from_visited: ??u8, + })); +} diff --git a/lib/std/zon/stringify.zig b/lib/std/zon/stringify.zig index 4cb2640063aa8870efed3abf21fdf701cb7ca9b8..785a303f22b69f304e78fb4f426f2c6282da4d94 100644 --- a/lib/std/zon/stringify.zig +++ b/lib/std/zon/stringify.zig @@ -23,13 +23,13 @@ const std = @import("std"); const assert = std.debug.assert; const Writer = std.Io.Writer; +const Serializer = std.zon.Serializer; -/// Options for `serialize`. pub const SerializeOptions = struct { /// If false, whitespace is omitted. Otherwise whitespace is emitted in standard Zig style. whitespace: bool = true, /// Determines when to emit Unicode code point literals as opposed to integer literals. - emit_codepoint_literals: EmitCodepointLiterals = .never, + emit_codepoint_literals: Serializer.EmitCodepointLiterals = .never, /// If true, slices of `u8`s, and pointers to arrays of `u8` are serialized as containers. /// Otherwise they are serialized as string literals. emit_strings_as_containers: bool = false, @@ -93,102 +93,6 @@ pub fn serializeArbitraryDepth( }); } -inline fn typeIsRecursive(comptime T: type) bool { - return comptime typeIsRecursiveInner(T, &.{}); -} - -fn typeIsRecursiveInner(comptime T: type, comptime prev_visited: []const type) bool { - for (prev_visited) |V| { - if (V == T) return true; - } - const visited = prev_visited ++ .{T}; - - return switch (@typeInfo(T)) { - .pointer => |pointer| typeIsRecursiveInner(pointer.child, visited), - .optional => |optional| typeIsRecursiveInner(optional.child, visited), - .array => |array| typeIsRecursiveInner(array.child, visited), - .vector => |vector| typeIsRecursiveInner(vector.child, visited), - .@"struct" => |@"struct"| for (@"struct".fields) |field| { - if (typeIsRecursiveInner(field.type, visited)) break true; - } else false, - .@"union" => |@"union"| inline for (@"union".fields) |field| { - if (typeIsRecursiveInner(field.type, visited)) break true; - } else false, - else => false, - }; -} - -inline fn canSerializeType(T: type) bool { - comptime return canSerializeTypeInner(T, &.{}, false); -} - -fn canSerializeTypeInner( - T: type, - /// Visited structs and unions, to avoid infinite recursion. - /// Tracking more types is unnecessary, and a little complex due to optional nesting. - visited: []const type, - parent_is_optional: bool, -) bool { - return switch (@typeInfo(T)) { - .bool, - .int, - .float, - .comptime_float, - .comptime_int, - .null, - .enum_literal, - => true, - - .noreturn, - .void, - .type, - .undefined, - .error_union, - .error_set, - .@"fn", - .frame, - .@"anyframe", - .@"opaque", - => false, - - .@"enum" => |@"enum"| @"enum".is_exhaustive, - - .pointer => |pointer| switch (pointer.size) { - .one => canSerializeTypeInner(pointer.child, visited, parent_is_optional), - .slice => canSerializeTypeInner(pointer.child, visited, false), - .many, .c => false, - }, - - .optional => |optional| if (parent_is_optional) - false - else - canSerializeTypeInner(optional.child, visited, true), - - .array => |array| canSerializeTypeInner(array.child, visited, false), - .vector => |vector| canSerializeTypeInner(vector.child, visited, false), - - .@"struct" => |@"struct"| { - for (visited) |V| if (T == V) return true; - const new_visited = visited ++ .{T}; - for (@"struct".fields) |field| { - if (!canSerializeTypeInner(field.type, new_visited, false)) return false; - } - return true; - }, - .@"union" => |@"union"| { - for (visited) |V| if (T == V) return true; - const new_visited = visited ++ .{T}; - if (@"union".tag_type == null) return false; - for (@"union".fields) |field| { - if (field.type != void and !canSerializeTypeInner(field.type, new_visited, false)) { - return false; - } - } - return true; - }, - }; -} - fn isNestedOptional(T: type) bool { comptime switch (@typeInfo(T)) { .optional => |optional| return isNestedOptionalInner(optional.child), @@ -210,842 +114,6 @@ fn isNestedOptionalInner(T: type) bool { } } -test "std.zon stringify canSerializeType" { - try std.testing.expect(!comptime canSerializeType(void)); - try std.testing.expect(!comptime canSerializeType(struct { f: [*]u8 })); - try std.testing.expect(!comptime canSerializeType(struct { error{foo} })); - try std.testing.expect(!comptime canSerializeType(union(enum) { a: void, f: [*c]u8 })); - try std.testing.expect(!comptime canSerializeType(@Vector(0, [*c]u8))); - try std.testing.expect(!comptime canSerializeType(*?[*c]u8)); - try std.testing.expect(!comptime canSerializeType(enum(u8) { _ })); - try std.testing.expect(!comptime canSerializeType(union { foo: void })); - try std.testing.expect(comptime canSerializeType(union(enum) { foo: void })); - try std.testing.expect(comptime canSerializeType(comptime_float)); - try std.testing.expect(comptime canSerializeType(comptime_int)); - try std.testing.expect(!comptime canSerializeType(struct { comptime foo: ??u8 = null })); - try std.testing.expect(comptime canSerializeType(@TypeOf(.foo))); - try std.testing.expect(comptime canSerializeType(?u8)); - try std.testing.expect(comptime canSerializeType(*?*u8)); - try std.testing.expect(comptime canSerializeType(?struct { - foo: ?struct { - ?union(enum) { - a: ?@Vector(0, ?*u8), - }, - ?struct { - f: ?[]?u8, - }, - }, - })); - try std.testing.expect(!comptime canSerializeType(??u8)); - try std.testing.expect(!comptime canSerializeType(?*?u8)); - try std.testing.expect(!comptime canSerializeType(*?*?*u8)); - try std.testing.expect(comptime canSerializeType(struct { x: comptime_int = 2 })); - try std.testing.expect(comptime canSerializeType(struct { x: comptime_float = 2 })); - try std.testing.expect(comptime canSerializeType(struct { comptime_int })); - try std.testing.expect(comptime canSerializeType(struct { comptime x: @TypeOf(.foo) = .foo })); - const Recursive = struct { foo: ?*@This() }; - try std.testing.expect(comptime canSerializeType(Recursive)); - - // Make sure we validate nested optional before we early out due to already having seen - // a type recursion! - try std.testing.expect(!comptime canSerializeType(struct { - add_to_visited: ?u8, - retrieve_from_visited: ??u8, - })); -} - -test "std.zon typeIsRecursive" { - try std.testing.expect(!typeIsRecursive(bool)); - try std.testing.expect(!typeIsRecursive(struct { x: i32, y: i32 })); - try std.testing.expect(!typeIsRecursive(struct { i32, i32 })); - try std.testing.expect(typeIsRecursive(struct { x: i32, y: i32, z: *@This() })); - try std.testing.expect(typeIsRecursive(struct { - a: struct { - const A = @This(); - b: struct { - c: *struct { - a: ?A, - }, - }, - }, - })); - try std.testing.expect(typeIsRecursive(struct { - a: [3]*@This(), - })); - try std.testing.expect(typeIsRecursive(struct { - a: union { a: i32, b: *@This() }, - })); -} - -fn checkValueDepth(val: anytype, depth: usize) error{ExceededMaxDepth}!void { - if (depth == 0) return error.ExceededMaxDepth; - const child_depth = depth - 1; - - switch (@typeInfo(@TypeOf(val))) { - .pointer => |pointer| switch (pointer.size) { - .one => try checkValueDepth(val.*, child_depth), - .slice => for (val) |item| { - try checkValueDepth(item, child_depth); - }, - .c, .many => {}, - }, - .array => for (val) |item| { - try checkValueDepth(item, child_depth); - }, - .@"struct" => |@"struct"| inline for (@"struct".fields) |field_info| { - try checkValueDepth(@field(val, field_info.name), child_depth); - }, - .@"union" => |@"union"| if (@"union".tag_type == null) { - return; - } else switch (val) { - inline else => |payload| { - return checkValueDepth(payload, child_depth); - }, - }, - .optional => if (val) |inner| try checkValueDepth(inner, child_depth), - else => {}, - } -} - -fn expectValueDepthEquals(expected: usize, value: anytype) !void { - try checkValueDepth(value, expected); - try std.testing.expectError(error.ExceededMaxDepth, checkValueDepth(value, expected - 1)); -} - -test "std.zon checkValueDepth" { - try expectValueDepthEquals(1, 10); - try expectValueDepthEquals(2, .{ .x = 1, .y = 2 }); - try expectValueDepthEquals(2, .{ 1, 2 }); - try expectValueDepthEquals(3, .{ 1, .{ 2, 3 } }); - try expectValueDepthEquals(3, .{ .{ 1, 2 }, 3 }); - try expectValueDepthEquals(3, .{ .x = 0, .y = 1, .z = .{ .x = 3 } }); - try expectValueDepthEquals(3, .{ .x = 0, .y = .{ .x = 1 }, .z = 2 }); - try expectValueDepthEquals(3, .{ .x = .{ .x = 0 }, .y = 1, .z = 2 }); - try expectValueDepthEquals(2, @as(?u32, 1)); - try expectValueDepthEquals(1, @as(?u32, null)); - try expectValueDepthEquals(1, null); - try expectValueDepthEquals(2, &1); - try expectValueDepthEquals(3, &@as(?u32, 1)); - - const Union = union(enum) { - x: u32, - y: struct { x: u32 }, - }; - try expectValueDepthEquals(2, Union{ .x = 1 }); - try expectValueDepthEquals(3, Union{ .y = .{ .x = 1 } }); - - const Recurse = struct { r: ?*const @This() }; - try expectValueDepthEquals(2, Recurse{ .r = null }); - try expectValueDepthEquals(5, Recurse{ .r = &Recurse{ .r = null } }); - try expectValueDepthEquals(8, Recurse{ .r = &Recurse{ .r = &Recurse{ .r = null } } }); - - try expectValueDepthEquals(2, @as([]const u8, &.{ 1, 2, 3 })); - try expectValueDepthEquals(3, @as([]const []const u8, &.{&.{ 1, 2, 3 }})); -} - -/// Determines when to emit Unicode code point literals as opposed to integer literals. -pub const EmitCodepointLiterals = enum { - /// Never emit Unicode code point literals. - never, - /// Emit Unicode code point literals for any `u8` in the printable ASCII range. - printable_ascii, - /// Emit Unicode code point literals for any unsigned integer with 21 bits or fewer - /// whose value is a valid non-surrogate code point. - always, - - /// If the value should be emitted as a Unicode codepoint, return it as a u21. - fn emitAsCodepoint(self: @This(), val: anytype) ?u21 { - // Rule out incompatible integer types - switch (@typeInfo(@TypeOf(val))) { - .int => |int_info| if (int_info.signedness == .signed or int_info.bits > 21) { - return null; - }, - .comptime_int => {}, - else => comptime unreachable, - } - - // Return null if the value shouldn't be printed as a Unicode codepoint, or the value casted - // to a u21 if it should. - switch (self) { - .always => { - const c = std.math.cast(u21, val) orelse return null; - if (!std.unicode.utf8ValidCodepoint(c)) return null; - return c; - }, - .printable_ascii => { - const c = std.math.cast(u8, val) orelse return null; - if (!std.ascii.isPrint(c)) return null; - return c; - }, - .never => { - return null; - }, - } - } -}; - -/// Options for serialization of an individual value. -/// -/// See `SerializeOptions` for more information on these options. -pub const ValueOptions = struct { - emit_codepoint_literals: EmitCodepointLiterals = .never, - emit_strings_as_containers: bool = false, - emit_default_optional_fields: bool = true, -}; - -/// Options for manual serialization of container types. -pub const SerializeContainerOptions = struct { - /// The whitespace style that should be used for this container. Ignored if whitespace is off. - whitespace_style: union(enum) { - /// If true, wrap every field. If false do not. - wrap: bool, - /// Automatically decide whether to wrap or not based on the number of fields. Following - /// the standard rule of thumb, containers with more than two fields are wrapped. - fields: usize, - } = .{ .wrap = true }, - - fn shouldWrap(self: SerializeContainerOptions) bool { - return switch (self.whitespace_style) { - .wrap => |wrap| wrap, - .fields => |fields| fields > 2, - }; - } -}; - -/// Lower level control over serialization, you can create a new instance with `serializer`. -/// -/// Useful when you want control over which fields are serialized, how they're represented, -/// or want to write a ZON object that does not exist in memory. -/// -/// You can serialize values with `value`. To serialize recursive types, the following are provided: -/// * `valueMaxDepth` -/// * `valueArbitraryDepth` -/// -/// You can also serialize values using specific notations: -/// * `int` -/// * `float` -/// * `codePoint` -/// * `tuple` -/// * `tupleMaxDepth` -/// * `tupleArbitraryDepth` -/// * `string` -/// * `multilineString` -/// -/// For manual serialization of containers, see: -/// * `beginStruct` -/// * `beginTuple` -pub const Serializer = struct { - options: Options = .{}, - indent_level: u8 = 0, - writer: *Writer, - - pub const Error = Writer.Error; - pub const DepthError = Error || error{ExceededMaxDepth}; - - pub const Options = struct { - /// If false, only syntactically necessary whitespace is emitted. - whitespace: bool = true, - }; - - /// Serialize a value, similar to `serialize`. - pub fn value(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - return self.valueArbitraryDepth(val, options); - } - - /// Serialize a value, similar to `serializeMaxDepth`. - /// Can return `error.ExceededMaxDepth`. - pub fn valueMaxDepth(self: *Serializer, val: anytype, options: ValueOptions, depth: usize) DepthError!void { - try checkValueDepth(val, depth); - return self.valueArbitraryDepth(val, options); - } - - /// Serialize a value, similar to `serializeArbitraryDepth`. - pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(canSerializeType(@TypeOf(val))); - switch (@typeInfo(@TypeOf(val))) { - .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { - self.codePoint(c) catch |err| switch (err) { - error.InvalidCodepoint => unreachable, // Already validated - else => |e| return e, - }; - } else { - try self.int(val); - }, - .float, .comptime_float => try self.float(val), - .bool, .null => try self.writer.print("{}", .{val}), - .enum_literal => try self.ident(@tagName(val)), - .@"enum" => try self.ident(@tagName(val)), - .pointer => |pointer| { - // Try to serialize as a string - const item: ?type = switch (@typeInfo(pointer.child)) { - .array => |array| array.child, - else => if (pointer.size == .slice) pointer.child else null, - }; - if (item == u8 and - (pointer.sentinel() == null or pointer.sentinel() == 0) and - !options.emit_strings_as_containers) - { - return try self.string(val); - } - - // Serialize as either a tuple or as the child type - switch (pointer.size) { - .slice => try self.tupleImpl(val, options), - .one => try self.valueArbitraryDepth(val.*, options), - else => comptime unreachable, - } - }, - .array => { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = val.len } }, - ); - for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - .@"struct" => |@"struct"| if (@"struct".is_tuple) { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = @"struct".fields.len } }, - ); - inline for (val) |field_value| { - try container.fieldArbitraryDepth(field_value, options); - } - try container.end(); - } else { - // Decide which fields to emit - const fields, const skipped: [@"struct".fields.len]bool = if (options.emit_default_optional_fields) b: { - break :b .{ @"struct".fields.len, @splat(false) }; - } else b: { - var fields = @"struct".fields.len; - var skipped: [@"struct".fields.len]bool = @splat(false); - inline for (@"struct".fields, &skipped) |field_info, *skip| { - if (field_info.default_value_ptr) |ptr| { - const default: *const field_info.type = @ptrCast(@alignCast(ptr)); - const field_value = @field(val, field_info.name); - if (std.meta.eql(field_value, default.*)) { - skip.* = true; - fields -= 1; - } - } - } - break :b .{ fields, skipped }; - }; - - // Emit those fields - var container = try self.beginStruct( - .{ .whitespace_style = .{ .fields = fields } }, - ); - inline for (@"struct".fields, skipped) |field_info, skip| { - if (!skip) { - try container.fieldArbitraryDepth( - field_info.name, - @field(val, field_info.name), - options, - ); - } - } - try container.end(); - }, - .@"union" => |@"union"| { - comptime assert(@"union".tag_type != null); - switch (val) { - inline else => |pl, tag| if (@TypeOf(pl) == void) - try self.writer.print(".{s}", .{@tagName(tag)}) - else { - var container = try self.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); - - try container.fieldArbitraryDepth( - @tagName(tag), - pl, - options, - ); - - try container.end(); - }, - } - }, - .optional => if (val) |inner| { - try self.valueArbitraryDepth(inner, options); - } else { - try self.writer.writeAll("null"); - }, - .vector => |vector| { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = vector.len } }, - ); - for (0..vector.len) |i| { - try container.fieldArbitraryDepth(val[i], options); - } - try container.end(); - }, - - else => comptime unreachable, - } - } - - /// Serialize an integer. - pub fn int(self: *Serializer, val: anytype) Error!void { - try self.writer.printInt(val, 10, .lower, .{}); - } - - /// Serialize a float. - pub fn float(self: *Serializer, val: anytype) Error!void { - switch (@typeInfo(@TypeOf(val))) { - .float => if (std.math.isNan(val)) { - return self.writer.writeAll("nan"); - } else if (std.math.isPositiveInf(val)) { - return self.writer.writeAll("inf"); - } else if (std.math.isNegativeInf(val)) { - return self.writer.writeAll("-inf"); - } else if (std.math.isNegativeZero(val)) { - return self.writer.writeAll("-0.0"); - } else { - try self.writer.print("{d}", .{val}); - }, - .comptime_float => if (val == 0) { - return self.writer.writeAll("0"); - } else { - try self.writer.print("{d}", .{val}); - }, - else => comptime unreachable, - } - } - - /// Serialize `name` as an identifier prefixed with `.`. - /// - /// Escapes the identifier if necessary. - pub fn ident(self: *Serializer, name: []const u8) Error!void { - try self.writer.print(".{f}", .{std.zig.fmtIdPU(name)}); - } - - pub const CodePointError = Error || error{InvalidCodepoint}; - - /// Serialize `val` as a Unicode codepoint. - /// - /// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint. - pub fn codePoint(self: *Serializer, val: u21) CodePointError!void { - try self.writer.print("'{f}'", .{std.zig.fmtChar(val)}); - } - - /// Like `value`, but always serializes `val` as a tuple. - /// - /// Will fail at comptime if `val` is not a tuple, array, pointer to an array, or slice. - pub fn tuple(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - try self.tupleArbitraryDepth(val, options); - } - - /// Like `tuple`, but recursive types are allowed. - /// - /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - pub fn tupleMaxDepth( - self: *Serializer, - val: anytype, - options: ValueOptions, - depth: usize, - ) DepthError!void { - try checkValueDepth(val, depth); - try self.tupleArbitraryDepth(val, options); - } - - /// Like `tuple`, but recursive types are allowed. - /// - /// It is the caller's responsibility to ensure that `val` does not contain cycles. - pub fn tupleArbitraryDepth( - self: *Serializer, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.tupleImpl(val, options); - } - - fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(canSerializeType(@TypeOf(val))); - switch (@typeInfo(@TypeOf(val))) { - .@"struct" => { - var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); - inline for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - .pointer, .array => { - var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); - for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - else => comptime unreachable, - } - } - - /// Like `value`, but always serializes `val` as a string. - pub fn string(self: *Serializer, val: []const u8) Error!void { - try self.writer.print("\"{f}\"", .{std.zig.fmtString(val)}); - } - - /// Options for formatting multiline strings. - pub const MultilineStringOptions = struct { - /// If top level is true, whitespace before and after the multiline string is elided. - /// If it is true, a newline is printed, then the value, followed by a newline, and if - /// whitespace is true any necessary indentation follows. - top_level: bool = false, - }; - - pub const MultilineStringError = Error || error{InnerCarriageReturn}; - - /// Like `value`, but always serializes to a multiline string literal. - /// - /// Returns `error.InnerCarriageReturn` if `val` contains a CR not followed by a newline, - /// since multiline strings cannot represent CR without a following newline. - pub fn multilineString( - self: *Serializer, - val: []const u8, - options: MultilineStringOptions, - ) MultilineStringError!void { - // Make sure the string does not contain any carriage returns not followed by a newline - var i: usize = 0; - while (i < val.len) : (i += 1) { - if (val[i] == '\r') { - if (i + 1 < val.len) { - if (val[i + 1] == '\n') { - i += 1; - continue; - } - } - return error.InnerCarriageReturn; - } - } - - if (!options.top_level) { - try self.newline(); - try self.indent(); - } - - try self.writer.writeAll("\\\\"); - for (val) |c| { - if (c != '\r') { - try self.writer.writeByte(c); // We write newlines here even if whitespace off - if (c == '\n') { - try self.indent(); - try self.writer.writeAll("\\\\"); - } - } - } - - if (!options.top_level) { - try self.writer.writeByte('\n'); // Even if whitespace off - try self.indent(); - } - } - - /// Create a `Struct` for writing ZON structs field by field. - pub fn beginStruct( - self: *Serializer, - options: SerializeContainerOptions, - ) Error!Struct { - return Struct.begin(self, options); - } - - /// Creates a `Tuple` for writing ZON tuples field by field. - pub fn beginTuple( - self: *Serializer, - options: SerializeContainerOptions, - ) Error!Tuple { - return Tuple.begin(self, options); - } - - fn indent(self: *Serializer) Error!void { - if (self.options.whitespace) { - try self.writer.splatByteAll(' ', 4 * self.indent_level); - } - } - - fn newline(self: *Serializer) Error!void { - if (self.options.whitespace) { - try self.writer.writeByte('\n'); - } - } - - fn newlineOrSpace(self: *Serializer, len: usize) Error!void { - if (self.containerShouldWrap(len)) { - try self.newline(); - } else { - try self.space(); - } - } - - fn space(self: *Serializer) Error!void { - if (self.options.whitespace) { - try self.writer.writeByte(' '); - } - } - - /// Writes ZON tuples field by field. - pub const Tuple = struct { - container: Container, - - fn begin(parent: *Serializer, options: SerializeContainerOptions) Error!Tuple { - return .{ - .container = try Container.begin(parent, .anon, options), - }; - } - - /// Finishes serializing the tuple. - /// - /// Prints a trailing comma as configured when appropriate, and the closing bracket. - pub fn end(self: *Tuple) Error!void { - try self.container.end(); - self.* = undefined; - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. - pub fn field( - self: *Tuple, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.container.field(null, val, options); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. - /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - pub fn fieldMaxDepth( - self: *Tuple, - val: anytype, - options: ValueOptions, - depth: usize, - ) DepthError!void { - try self.container.fieldMaxDepth(null, val, options, depth); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by - /// `valueArbitraryDepth`. - pub fn fieldArbitraryDepth( - self: *Tuple, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.container.fieldArbitraryDepth(null, val, options); - } - - /// Starts a field with a struct as a value. Returns the struct. - pub fn beginStructField( - self: *Tuple, - options: SerializeContainerOptions, - ) Error!Struct { - try self.fieldPrefix(); - return self.container.serializer.beginStruct(options); - } - - /// Starts a field with a tuple as a value. Returns the tuple. - pub fn beginTupleField( - self: *Tuple, - options: SerializeContainerOptions, - ) Error!Tuple { - try self.fieldPrefix(); - return self.container.serializer.beginTuple(options); - } - - /// Print a field prefix. This prints any necessary commas, and whitespace as - /// configured. Useful if you want to serialize the field value yourself. - pub fn fieldPrefix(self: *Tuple) Error!void { - try self.container.fieldPrefix(null); - } - }; - - /// Writes ZON structs field by field. - pub const Struct = struct { - container: Container, - - fn begin(parent: *Serializer, options: SerializeContainerOptions) Error!Struct { - return .{ - .container = try Container.begin(parent, .named, options), - }; - } - - /// Finishes serializing the struct. - /// - /// Prints a trailing comma as configured when appropriate, and the closing bracket. - pub fn end(self: *Struct) Error!void { - try self.container.end(); - self.* = undefined; - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. - pub fn field( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.container.field(name, val, options); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. - /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - pub fn fieldMaxDepth( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - depth: usize, - ) DepthError!void { - try self.container.fieldMaxDepth(name, val, options, depth); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by - /// `valueArbitraryDepth`. - pub fn fieldArbitraryDepth( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.container.fieldArbitraryDepth(name, val, options); - } - - /// Starts a field with a struct as a value. Returns the struct. - pub fn beginStructField( - self: *Struct, - name: []const u8, - options: SerializeContainerOptions, - ) Error!Struct { - try self.fieldPrefix(name); - return self.container.serializer.beginStruct(options); - } - - /// Starts a field with a tuple as a value. Returns the tuple. - pub fn beginTupleField( - self: *Struct, - name: []const u8, - options: SerializeContainerOptions, - ) Error!Tuple { - try self.fieldPrefix(name); - return self.container.serializer.beginTuple(options); - } - - /// Print a field prefix. This prints any necessary commas, the field name (escaped if - /// necessary) and whitespace as configured. Useful if you want to serialize the field - /// value yourself. - pub fn fieldPrefix(self: *Struct, name: []const u8) Error!void { - try self.container.fieldPrefix(name); - } - }; - - const Container = struct { - const FieldStyle = enum { named, anon }; - - serializer: *Serializer, - field_style: FieldStyle, - options: SerializeContainerOptions, - empty: bool, - - fn begin( - sz: *Serializer, - field_style: FieldStyle, - options: SerializeContainerOptions, - ) Error!Container { - if (options.shouldWrap()) sz.indent_level +|= 1; - try sz.writer.writeAll(".{"); - return .{ - .serializer = sz, - .field_style = field_style, - .options = options, - .empty = true, - }; - } - - fn end(self: *Container) Error!void { - if (self.options.shouldWrap()) self.serializer.indent_level -|= 1; - if (!self.empty) { - if (self.options.shouldWrap()) { - if (self.serializer.options.whitespace) { - try self.serializer.writer.writeByte(','); - } - try self.serializer.newline(); - try self.serializer.indent(); - } else if (!self.shouldElideSpaces()) { - try self.serializer.space(); - } - } - try self.serializer.writer.writeByte('}'); - self.* = undefined; - } - - fn fieldPrefix(self: *Container, name: ?[]const u8) Error!void { - if (!self.empty) { - try self.serializer.writer.writeByte(','); - } - self.empty = false; - if (self.options.shouldWrap()) { - try self.serializer.newline(); - } else if (!self.shouldElideSpaces()) { - try self.serializer.space(); - } - if (self.options.shouldWrap()) try self.serializer.indent(); - if (name) |n| { - try self.serializer.ident(n); - try self.serializer.space(); - try self.serializer.writer.writeByte('='); - try self.serializer.space(); - } - } - - fn field( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - ) Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - try self.fieldArbitraryDepth(name, val, options); - } - - /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - fn fieldMaxDepth( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - depth: usize, - ) DepthError!void { - try checkValueDepth(val, depth); - try self.fieldArbitraryDepth(name, val, options); - } - - fn fieldArbitraryDepth( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.fieldPrefix(name); - try self.serializer.valueArbitraryDepth(val, options); - } - - fn shouldElideSpaces(self: *const Container) bool { - return switch (self.options.whitespace_style) { - .fields => |fields| self.field_style != .named and fields == 1, - else => false, - }; - } - }; -}; - -test Serializer { - var discarding: Writer.Discarding = .init(&.{}); - var s: Serializer = .{ .writer = &discarding.writer }; - var vec2 = try s.beginStruct(.{}); - try vec2.field("x", 1.5, .{}); - try vec2.fieldPrefix("prefix"); - try s.value(2.5, .{}); - try vec2.end(); -} - fn expectSerializeEqual( expected: []const u8, value: anytype, diff --git a/src/main.zig b/src/main.zig index 019ae8bc7e8885a77c9c1ca0b7b1205af1b74be7..2eb5762a0c156a1713f3dc777c634fcf21af5cc6 100644 --- a/src/main.zig +++ b/src/main.zig @@ -344,8 +344,9 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "targets")) { dev.check(.targets_command); const host = std.zig.resolveTargetQueryOrFatal(.{}); - const stdout = fs.File.stdout().deprecatedWriter(); - return @import("print_targets.zig").cmdTargets(arena, cmd_args, stdout, &host); + var stdout_writer = fs.File.stdout().writer(&stdio_buffer); + try @import("print_targets.zig").cmdTargets(arena, cmd_args, &stdout_writer.interface, &host); + return stdout_writer.interface.flush(); } else if (mem.eql(u8, cmd, "version")) { dev.check(.version_command); try fs.File.stdout().writeAll(build_options.version ++ "\n"); @@ -356,7 +357,9 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "env")) { dev.check(.env_command); verifyLibcxxCorrectlyLinked(); - return @import("print_env.zig").cmdEnv(arena, cmd_args); + var stdout_writer = fs.File.stdout().writer(&stdio_buffer); + try @import("print_env.zig").cmdEnv(arena, &stdout_writer.interface); + return stdout_writer.interface.flush(); } else if (mem.eql(u8, cmd, "reduce")) { return jitCmd(gpa, arena, cmd_args, .{ .cmd_name = "reduce", diff --git a/src/print_env.zig b/src/print_env.zig index d1546abc6c64196031d9c6dcb43b0c6fc836fafa..d1251c0d62c89f4aaa950f98d520bbae2983b980 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -4,8 +4,7 @@ const introspect = @import("introspect.zig"); const Allocator = std.mem.Allocator; const fatal = std.process.fatal; -pub fn cmdEnv(arena: Allocator, args: []const []const u8) !void { - _ = args; +pub fn cmdEnv(arena: Allocator, out: *std.Io.Writer) !void { const cwd_path = try introspect.getResolvedCwd(arena); const self_exe_path = try std.fs.selfExePathAlloc(arena); @@ -21,41 +20,21 @@ pub fn cmdEnv(arena: Allocator, args: []const []const u8) !void { const host = try std.zig.system.resolveTargetQuery(.{}); const triple = try host.zigTriple(arena); - var buffer: [1024]u8 = undefined; - var stdout_writer = std.fs.File.stdout().writer(&buffer); - - var jws: std.json.Stringify = .{ .writer = &stdout_writer.interface, .options = .{ .whitespace = .indent_1 } }; - - try jws.beginObject(); - - try jws.objectField("zig_exe"); - try jws.write(self_exe_path); - - try jws.objectField("lib_dir"); - try jws.write(zig_lib_directory.path.?); - - try jws.objectField("std_dir"); - try jws.write(zig_std_dir); - - try jws.objectField("global_cache_dir"); - try jws.write(global_cache_dir); - - try jws.objectField("version"); - try jws.write(build_options.version); - - try jws.objectField("target"); - try jws.write(triple); - - try jws.objectField("env"); - try jws.beginObject(); + var serializer: std.zon.Serializer = .{ .writer = out }; + var root = try serializer.beginStruct(.{}); + + try root.field("zig_exe", self_exe_path, .{}); + try root.field("lib_dir", zig_lib_directory.path.?, .{}); + try root.field("std_dir", zig_std_dir, .{}); + try root.field("global_cache_dir", global_cache_dir, .{}); + try root.field("version", build_options.version, .{}); + try root.field("target", triple, .{}); + var env = try root.beginStructField("env", .{}); inline for (@typeInfo(std.zig.EnvVar).@"enum".fields) |field| { - try jws.objectField(field.name); - try jws.write(try @field(std.zig.EnvVar, field.name).get(arena)); + try env.field(field.name, try @field(std.zig.EnvVar, field.name).get(arena), .{}); } - try jws.endObject(); + try env.end(); + try root.end(); - try jws.endObject(); - - try stdout_writer.interface.writeByte('\n'); - try stdout_writer.interface.flush(); + try out.writeByte('\n'); } diff --git a/src/print_targets.zig b/src/print_targets.zig index e234aeda26549d51684926b6d863255b526a1b7b..e04842bb7e16f73cc1d6cad4207c14ec7e898d62 100644 --- a/src/print_targets.zig +++ b/src/print_targets.zig @@ -14,8 +14,7 @@ const introspect = @import("introspect.zig"); pub fn cmdTargets( allocator: Allocator, args: []const []const u8, - /// Output stream - stdout: anytype, + out: *std.Io.Writer, native_target: *const Target, ) !void { _ = args; @@ -38,12 +37,10 @@ pub fn cmdTargets( const glibc_abi = try glibc.loadMetaData(allocator, abilists_contents); defer glibc_abi.destroy(allocator); - var bw = io.bufferedWriter(stdout); - const w = bw.writer(); - var sz = std.zon.stringify.serializer(w, .{}); + var serializer: std.zon.Serializer = .{ .writer = out }; { - var root_obj = try sz.beginStruct(.{}); + var root_obj = try serializer.beginStruct(.{}); try root_obj.field("arch", meta.fieldNames(Target.Cpu.Arch), .{}); try root_obj.field("os", meta.fieldNames(Target.Os.Tag), .{}); @@ -136,6 +133,5 @@ pub fn cmdTargets( try root_obj.end(); } - try w.writeByte('\n'); - return bw.flush(); + try out.writeByte('\n'); } diff --git a/src/translate_c.zig b/src/translate_c.zig index 301e0a219d18a01c1440fc1f9ef8721c5b500224..f1f3ad865946c4d045f236af530e3e1861639588 100644 --- a/src/translate_c.zig +++ b/src/translate_c.zig @@ -3338,7 +3338,7 @@ fn transPredefinedExpr(c: *Context, scope: *Scope, expr: *const clang.Predefined fn transCreateCharLitNode(c: *Context, narrow: bool, val: u32) TransError!Node { return Tag.char_literal.create(c.arena, if (narrow) - try std.fmt.allocPrint(c.arena, "'{f}'", .{std.zig.fmtChar(&.{@as(u8, @intCast(val))})}) + try std.fmt.allocPrint(c.arena, "'{f}'", .{std.zig.fmtChar(@intCast(val))}) else try std.fmt.allocPrint(c.arena, "'\\u{{{x}}}'", .{val})); } -- 2.54.0 From 592f1043dc704faf79bde7149917d4155b3747f8 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sat, 19 Jul 2025 13:54:31 -0400 Subject: [PATCH 10/26] cbe: fix comptime-known packed unions --- src/codegen/c.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 955a83bc3336c90b17fe2fca38b4639f350815dd..c726c05e1b30085ba74fa19527bc29ed12b57bcb 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -2438,7 +2438,10 @@ pub const DeclGen = struct { const ty = val.typeOf(zcu); return .{ .data = .{ .dg = dg, - .int_info = ty.intInfo(zcu), + .int_info = if (ty.zigTypeTag(zcu) == .@"union" and ty.containerLayout(zcu) == .@"packed") + .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) } + else + ty.intInfo(zcu), .kind = kind, .ctype = try dg.ctypeFromType(ty, kind), .val = val, -- 2.54.0 From 93378e2e7be964a7362a9875e3292e9d6a697177 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Jul 2025 19:47:58 -0700 Subject: [PATCH 11/26] std.zig: finish updating to new I/O API --- lib/compiler/objcopy.zig | 18 +- lib/compiler/resinator/main.zig | 6 +- lib/compiler/test_runner.zig | 11 +- lib/std/zig.zig | 1 + lib/std/zig/LibCInstallation.zig | 7 +- lib/std/zig/Server.zig | 214 +++++++---------------- lib/std/zig/WindowsSdk.zig | 14 +- lib/std/zig/llvm.zig | 6 + lib/std/zig/llvm/BitcodeReader.zig | 28 +-- lib/std/zig/perf_test.zig | 14 +- lib/std/zig/system/linux.zig | 35 ++-- src/Compilation.zig | 85 +++++----- src/deprecated.zig | 262 ----------------------------- src/main.zig | 46 ++--- 14 files changed, 201 insertions(+), 546 deletions(-) diff --git a/lib/compiler/objcopy.zig b/lib/compiler/objcopy.zig index 98bf5d2bcfe0220d398aad1ab8e8ab8645bd2a34..52ffe208f6219f1c8973753af8a5e9b44c4d89d4 100644 --- a/lib/compiler/objcopy.zig +++ b/lib/compiler/objcopy.zig @@ -10,6 +10,9 @@ const assert = std.debug.assert; const fatal = std.process.fatal; const Server = std.zig.Server; +var stdin_buffer: [1024]u8 = undefined; +var stdout_buffer: [1024]u8 = undefined; + pub fn main() !void { var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena_instance.deinit(); @@ -22,11 +25,8 @@ pub fn main() !void { return cmdObjCopy(gpa, arena, args[1..]); } -fn cmdObjCopy( - gpa: Allocator, - arena: Allocator, - args: []const []const u8, -) !void { +fn cmdObjCopy(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { + _ = gpa; var i: usize = 0; var opt_out_fmt: ?std.Target.ObjectFormat = null; var opt_input: ?[]const u8 = null; @@ -225,13 +225,13 @@ fn cmdObjCopy( } if (listen) { + var stdin_reader = fs.File.stdin().reader(&stdin_buffer); + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); var server = try Server.init(.{ - .gpa = gpa, - .in = .stdin(), - .out = .stdout(), + .in = &stdin_reader.interface, + .out = &stdout_writer.interface, .zig_version = builtin.zig_version_string, }); - defer server.deinit(); var seen_update = false; while (true) { diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index 903e0a2f719a3beb20e80c07e7b08814a988954c..4c952c03c4a46653daddb9903d3f927ff0f28653 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -13,6 +13,8 @@ const hasDisjointCodePage = @import("disjoint_code_page.zig").hasDisjointCodePag const fmtResourceType = @import("res.zig").NameOrOrdinal.fmtResourceType; const aro = @import("aro"); +var stdout_buffer: [1024]u8 = undefined; + pub fn main() !void { var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; defer std.debug.assert(gpa.deinit() == .ok); @@ -41,12 +43,12 @@ pub fn main() !void { cli_args = args[3..]; } + var stdout_writer2 = std.fs.File.stdout().writer(&stdout_buffer); var error_handler: ErrorHandler = switch (zig_integration) { true => .{ .server = .{ - .out = std.fs.File.stdout(), + .out = &stdout_writer2.interface, .in = undefined, // won't be receiving messages - .receive_fifo = undefined, // won't be receiving messages }, }, false => .{ diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 929bd1c417f933a8aa8438143ff70644b2019427..a69066f09cb9f2de19bd3c8cb4ad6fef63ef0fe2 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -2,7 +2,6 @@ const builtin = @import("builtin"); const std = @import("std"); -const io = std.io; const testing = std.testing; const assert = std.debug.assert; @@ -13,6 +12,8 @@ pub const std_options: std.Options = .{ var log_err_count: usize = 0; var fba_buffer: [8192]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); +var stdin_buffer: [std.heap.page_size_min]u8 align(std.heap.page_size_min) = undefined; +var stdout_buffer: [std.heap.page_size_min]u8 align(std.heap.page_size_min) = undefined; const crippled = switch (builtin.zig_backend) { .stage2_powerpc, @@ -67,13 +68,13 @@ pub fn main() void { fn mainServer() !void { @disableInstrumentation(); + var stdin_reader = std.fs.File.stdin().reader(&stdin_buffer); + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); var server = try std.zig.Server.init(.{ - .gpa = fba.allocator(), - .in = .stdin(), - .out = .stdout(), + .in = &stdin_reader.interface, + .out = &stdout_writer.interface, .zig_version = builtin.zig_version_string, }); - defer server.deinit(); if (builtin.fuzz) { const coverage_id = fuzzer_coverage_id(); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 51eac1b0a6fa09855730c2d074421f0afda22198..ad264a9b3377341285119ef4e953ca4bf40f0cd0 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -905,4 +905,5 @@ test { _ = system; _ = target; _ = c_translation; + _ = llvm; } diff --git a/lib/std/zig/LibCInstallation.zig b/lib/std/zig/LibCInstallation.zig index e2b7cd233ddba0c015fa9c88b02d2296ef287208..3e03d1ff2e747bd6ba46ca1d06b65f4c3eba7e3c 100644 --- a/lib/std/zig/LibCInstallation.zig +++ b/lib/std/zig/LibCInstallation.zig @@ -370,7 +370,7 @@ fn findNativeIncludeDirWindows( for (installs) |install| { result_buf.shrinkAndFree(0); - try result_buf.writer().print("{s}\\Include\\{s}\\ucrt", .{ install.path, install.version }); + try result_buf.print("{s}\\Include\\{s}\\ucrt", .{ install.path, install.version }); var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) { error.FileNotFound, @@ -417,7 +417,7 @@ fn findNativeCrtDirWindows( for (installs) |install| { result_buf.shrinkAndFree(0); - try result_buf.writer().print("{s}\\Lib\\{s}\\ucrt\\{s}", .{ install.path, install.version, arch_sub_dir }); + try result_buf.print("{s}\\Lib\\{s}\\ucrt\\{s}", .{ install.path, install.version, arch_sub_dir }); var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) { error.FileNotFound, @@ -484,8 +484,7 @@ fn findNativeKernel32LibDir( for (installs) |install| { result_buf.shrinkAndFree(0); - const stream = result_buf.writer(); - try stream.print("{s}\\Lib\\{s}\\um\\{s}", .{ install.path, install.version, arch_sub_dir }); + try result_buf.print("{s}\\Lib\\{s}\\um\\{s}", .{ install.path, install.version, arch_sub_dir }); var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) { error.FileNotFound, diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 2f03c7808318866c62b07dfdf14050c037b51ae1..38ad45e1e161acb6af9fc4669653695d78c37031 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -1,6 +1,20 @@ -in: std.fs.File, -out: std.fs.File, -receive_fifo: std.fifo.LinearFifo(u8, .Dynamic), +const Server = @This(); + +const builtin = @import("builtin"); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const native_endian = builtin.target.cpu.arch.endian(); +const need_bswap = native_endian != .little; +const Cache = std.Build.Cache; +const OutMessage = std.zig.Server.Message; +const InMessage = std.zig.Client.Message; +const Reader = std.Io.Reader; +const Writer = std.Io.Writer; + +in: *Reader, +out: *Writer, pub const Message = struct { pub const Header = extern struct { @@ -94,9 +108,8 @@ pub const Message = struct { }; pub const Options = struct { - gpa: Allocator, - in: std.fs.File, - out: std.fs.File, + in: *Reader, + out: *Writer, zig_version: []const u8, }; @@ -104,96 +117,40 @@ pub fn init(options: Options) !Server { var s: Server = .{ .in = options.in, .out = options.out, - .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa), }; try s.serveStringMessage(.zig_version, options.zig_version); return s; } -pub fn deinit(s: *Server) void { - s.receive_fifo.deinit(); - s.* = undefined; -} - pub fn receiveMessage(s: *Server) !InMessage.Header { - const Header = InMessage.Header; - const fifo = &s.receive_fifo; - var last_amt_zero = false; - - while (true) { - const buf = fifo.readableSlice(0); - assert(fifo.readableLength() == buf.len); - if (buf.len >= @sizeOf(Header)) { - const header: *align(1) const Header = @ptrCast(buf[0..@sizeOf(Header)]); - const bytes_len = bswap(header.bytes_len); - const tag = bswap(header.tag); - - if (buf.len - @sizeOf(Header) >= bytes_len) { - fifo.discard(@sizeOf(Header)); - return .{ - .tag = tag, - .bytes_len = bytes_len, - }; - } else { - const needed = bytes_len - (buf.len - @sizeOf(Header)); - const write_buffer = try fifo.writableWithSize(needed); - const amt = try s.in.read(write_buffer); - fifo.update(amt); - continue; - } - } - - const write_buffer = try fifo.writableWithSize(256); - const amt = try s.in.read(write_buffer); - fifo.update(amt); - if (amt == 0) { - if (last_amt_zero) return error.BrokenPipe; - last_amt_zero = true; - } - } + return s.in.takeStruct(InMessage.Header, .little); } pub fn receiveBody_u32(s: *Server) !u32 { - const fifo = &s.receive_fifo; - const buf = fifo.readableSlice(0); - const result = @as(*align(1) const u32, @ptrCast(buf[0..4])).*; - fifo.discard(4); - return bswap(result); + return s.in.takeInt(u32, .little); } pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void { - return s.serveMessage(.{ + try s.serveMessageHeader(.{ .tag = tag, - .bytes_len = @as(u32, @intCast(msg.len)), - }, &.{msg}); + .bytes_len = @intCast(msg.len), + }); + try s.out.writeAll(msg); + try s.out.flush(); } -pub fn serveMessage( - s: *const Server, - header: OutMessage.Header, - bufs: []const []const u8, -) !void { - var iovecs: [10]std.posix.iovec_const = undefined; - const header_le = bswap(header); - iovecs[0] = .{ - .base = @as([*]const u8, @ptrCast(&header_le)), - .len = @sizeOf(OutMessage.Header), - }; - for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { - iovec.* = .{ - .base = buf.ptr, - .len = buf.len, - }; - } - try s.out.writevAll(iovecs[0 .. bufs.len + 1]); +/// Don't forget to flush! +pub fn serveMessageHeader(s: *const Server, header: OutMessage.Header) !void { + try s.out.writeStruct(header, .little); } -pub fn serveU64Message(s: *Server, tag: OutMessage.Tag, int: u64) !void { - const msg_le = bswap(int); - return s.serveMessage(.{ +pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void { + try serveMessageHeader(s, .{ .tag = tag, .bytes_len = @sizeOf(u64), - }, &.{std.mem.asBytes(&msg_le)}); + }); + try s.out.writeInt(u64, int, .little); + try s.out.flush(); } pub fn serveEmitDigest( @@ -201,26 +158,22 @@ pub fn serveEmitDigest( digest: *const [Cache.bin_digest_len]u8, header: OutMessage.EmitDigest, ) !void { - try s.serveMessage(.{ + try s.serveMessageHeader(.{ .tag = .emit_digest, .bytes_len = @intCast(digest.len + @sizeOf(OutMessage.EmitDigest)), - }, &.{ - std.mem.asBytes(&header), - digest, }); + try s.out.writeStruct(header, .little); + try s.out.writeAll(digest); + try s.out.flush(); } -pub fn serveTestResults( - s: *Server, - msg: OutMessage.TestResults, -) !void { - const msg_le = bswap(msg); - try s.serveMessage(.{ +pub fn serveTestResults(s: *Server, msg: OutMessage.TestResults) !void { + try s.serveMessageHeader(.{ .tag = .test_results, .bytes_len = @intCast(@sizeOf(OutMessage.TestResults)), - }, &.{ - std.mem.asBytes(&msg_le), }); + try s.out.writeStruct(msg, .little); + try s.out.flush(); } pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { @@ -230,91 +183,38 @@ pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { }; const bytes_len = @sizeOf(OutMessage.ErrorBundle) + 4 * error_bundle.extra.len + error_bundle.string_bytes.len; - try s.serveMessage(.{ + try s.serveMessageHeader(.{ .tag = .error_bundle, .bytes_len = @intCast(bytes_len), - }, &.{ - std.mem.asBytes(&eb_hdr), - // TODO: implement @ptrCast between slices changing the length - std.mem.sliceAsBytes(error_bundle.extra), - error_bundle.string_bytes, }); + try s.out.writeStruct(eb_hdr, .little); + try s.out.writeSliceEndian(u32, error_bundle.extra, .little); + try s.out.writeAll(error_bundle.string_bytes); + try s.out.flush(); } pub const TestMetadata = struct { - names: []u32, - expected_panic_msgs: []u32, + names: []const u32, + expected_panic_msgs: []const u32, string_bytes: []const u8, }; pub fn serveTestMetadata(s: *Server, test_metadata: TestMetadata) !void { const header: OutMessage.TestMetadata = .{ - .tests_len = bswap(@as(u32, @intCast(test_metadata.names.len))), - .string_bytes_len = bswap(@as(u32, @intCast(test_metadata.string_bytes.len))), + .tests_len = @as(u32, @intCast(test_metadata.names.len)), + .string_bytes_len = @as(u32, @intCast(test_metadata.string_bytes.len)), }; const trailing = 2; const bytes_len = @sizeOf(OutMessage.TestMetadata) + trailing * @sizeOf(u32) * test_metadata.names.len + test_metadata.string_bytes.len; - if (need_bswap) { - bswap_u32_array(test_metadata.names); - bswap_u32_array(test_metadata.expected_panic_msgs); - } - defer if (need_bswap) { - bswap_u32_array(test_metadata.names); - bswap_u32_array(test_metadata.expected_panic_msgs); - }; - - return s.serveMessage(.{ + try s.serveMessageHeader(.{ .tag = .test_metadata, .bytes_len = @intCast(bytes_len), - }, &.{ - std.mem.asBytes(&header), - // TODO: implement @ptrCast between slices changing the length - std.mem.sliceAsBytes(test_metadata.names), - std.mem.sliceAsBytes(test_metadata.expected_panic_msgs), - test_metadata.string_bytes, }); + try s.out.writeStruct(header, .little); + try s.out.writeSliceEndian(u32, test_metadata.names, .little); + try s.out.writeSliceEndian(u32, test_metadata.expected_panic_msgs, .little); + try s.out.writeAll(test_metadata.string_bytes); + try s.out.flush(); } - -fn bswap(x: anytype) @TypeOf(x) { - if (!need_bswap) return x; - - const T = @TypeOf(x); - switch (@typeInfo(T)) { - .@"enum" => return @as(T, @enumFromInt(@byteSwap(@intFromEnum(x)))), - .int => return @byteSwap(x), - .@"struct" => |info| switch (info.layout) { - .@"extern" => { - var result: T = undefined; - inline for (info.fields) |field| { - @field(result, field.name) = bswap(@field(x, field.name)); - } - return result; - }, - .@"packed" => { - const I = info.backing_integer.?; - return @as(T, @bitCast(@byteSwap(@as(I, @bitCast(x))))); - }, - .auto => @compileError("auto layout struct"), - }, - else => @compileError("bswap on type " ++ @typeName(T)), - } -} - -fn bswap_u32_array(slice: []u32) void { - comptime assert(need_bswap); - for (slice) |*elem| elem.* = @byteSwap(elem.*); -} - -const OutMessage = std.zig.Server.Message; -const InMessage = std.zig.Client.Message; - -const Server = @This(); -const builtin = @import("builtin"); -const std = @import("std"); -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const native_endian = builtin.target.cpu.arch.endian(); -const need_bswap = native_endian != .little; -const Cache = std.Build.Cache; diff --git a/lib/std/zig/WindowsSdk.zig b/lib/std/zig/WindowsSdk.zig index 1bcf45fb7451eb56ecc330016f440c558772e0d4..61e1defb129068afa88c7ad5b829eba2a426a2e5 100644 --- a/lib/std/zig/WindowsSdk.zig +++ b/lib/std/zig/WindowsSdk.zig @@ -1,11 +1,12 @@ +const WindowsSdk = @This(); +const builtin = @import("builtin"); +const std = @import("std"); +const Writer = std.Io.Writer; + windows10sdk: ?Installation, windows81sdk: ?Installation, msvc_lib_dir: ?[]const u8, -const WindowsSdk = @This(); -const std = @import("std"); -const builtin = @import("builtin"); - const windows = std.os.windows; const RRF = windows.advapi32.RRF; @@ -759,14 +760,13 @@ const MsvcLibDir = struct { while (instances_dir_it.next() catch return error.PathNotFound) |entry| { if (entry.kind != .directory) continue; - var fbs = std.io.fixedBufferStream(&state_subpath_buf); - const writer = fbs.writer(); + var writer: Writer = .fixed(&state_subpath_buf); writer.writeAll(entry.name) catch unreachable; writer.writeByte(std.fs.path.sep) catch unreachable; writer.writeAll("state.json") catch unreachable; - const json_contents = instances_dir.readFileAlloc(allocator, fbs.getWritten(), std.math.maxInt(usize)) catch continue; + const json_contents = instances_dir.readFileAlloc(allocator, writer.buffered(), std.math.maxInt(usize)) catch continue; defer allocator.free(json_contents); var parsed = std.json.parseFromSlice(std.json.Value, allocator, json_contents, .{}) catch continue; diff --git a/lib/std/zig/llvm.zig b/lib/std/zig/llvm.zig index c2e1ed9c56bfd5a93b8aaaded80023e635dbe568..c45ffe90836cf487909236a410e21f6b52e2b503 100644 --- a/lib/std/zig/llvm.zig +++ b/lib/std/zig/llvm.zig @@ -1,3 +1,9 @@ pub const BitcodeReader = @import("llvm/BitcodeReader.zig"); pub const bitcode_writer = @import("llvm/bitcode_writer.zig"); pub const Builder = @import("llvm/Builder.zig"); + +test { + _ = BitcodeReader; + _ = bitcode_writer; + _ = Builder; +} diff --git a/lib/std/zig/llvm/BitcodeReader.zig b/lib/std/zig/llvm/BitcodeReader.zig index ea18b7978eb6aeff00bc45a7e82758ea4df16e7a..f1d47e93c0139c9a16f5a994e96918f8a564d0c5 100644 --- a/lib/std/zig/llvm/BitcodeReader.zig +++ b/lib/std/zig/llvm/BitcodeReader.zig @@ -1,6 +1,11 @@ +const BitcodeReader = @This(); + +const std = @import("../../std.zig"); +const assert = std.debug.assert; + allocator: std.mem.Allocator, record_arena: std.heap.ArenaAllocator.State, -reader: std.io.AnyReader, +reader: *std.Io.Reader, keep_names: bool, bit_buffer: u32, bit_offset: u5, @@ -93,7 +98,7 @@ pub const Record = struct { }; pub const InitOptions = struct { - reader: std.io.AnyReader, + reader: *std.Io.Reader, keep_names: bool = false, }; pub fn init(allocator: std.mem.Allocator, options: InitOptions) BitcodeReader { @@ -172,7 +177,7 @@ pub fn next(bc: *BitcodeReader) !?Item { pub fn skipBlock(bc: *BitcodeReader, block: Block) !void { assert(bc.bit_offset == 0); - try bc.reader.skipBytes(@as(u34, block.len) * 4, .{}); + try bc.reader.discardAll(4 * @as(u34, block.len)); try bc.endBlock(); } @@ -371,19 +376,19 @@ fn align32Bits(bc: *BitcodeReader) void { fn read32Bits(bc: *BitcodeReader) !u32 { assert(bc.bit_offset == 0); - return bc.reader.readInt(u32, .little); + return bc.reader.takeInt(u32, .little); } fn readBytes(bc: *BitcodeReader, bytes: []u8) !void { assert(bc.bit_offset == 0); - try bc.reader.readNoEof(bytes); + try bc.reader.readSliceAll(bytes); const trailing_bytes = bytes.len % 4; if (trailing_bytes > 0) { - var bit_buffer = [1]u8{0} ** 4; - try bc.reader.readNoEof(bit_buffer[trailing_bytes..]); + var bit_buffer: [4]u8 = @splat(0); + try bc.reader.readSliceAll(bit_buffer[trailing_bytes..]); bc.bit_buffer = std.mem.readInt(u32, &bit_buffer, .little); - bc.bit_offset = @intCast(trailing_bytes * 8); + bc.bit_offset = @intCast(8 * trailing_bytes); } } @@ -509,7 +514,6 @@ const Abbrev = struct { }; }; -const assert = std.debug.assert; -const std = @import("../../std.zig"); - -const BitcodeReader = @This(); +test { + _ = &skipBlock; +} diff --git a/lib/std/zig/perf_test.zig b/lib/std/zig/perf_test.zig index 087b081475594cf07106688ec3e6f379f99ab299..1566a15d2dc141f99e488e3309977220ed444f6e 100644 --- a/lib/std/zig/perf_test.zig +++ b/lib/std/zig/perf_test.zig @@ -1,7 +1,6 @@ const std = @import("std"); const mem = std.mem; const Tokenizer = std.zig.Tokenizer; -const io = std.io; const fmtIntSizeBin = std.fmt.fmtIntSizeBin; const source = @embedFile("../os.zig"); @@ -22,16 +21,15 @@ pub fn main() !void { const bytes_per_sec_float = @as(f64, @floatFromInt(source.len * iterations)) / elapsed_s; const bytes_per_sec = @as(u64, @intFromFloat(@floor(bytes_per_sec_float))); - var stdout_file: std.fs.File = .stdout(); - const stdout = stdout_file.deprecatedWriter(); - try stdout.print("parsing speed: {:.2}/s, {:.2} used \n", .{ - fmtIntSizeBin(bytes_per_sec), - fmtIntSizeBin(memory_used), - }); + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; + try stdout.print("parsing speed: {Bi:.2}/s, {Bi:.2} used \n", .{ bytes_per_sec, memory_used }); + try stdout.flush(); } fn testOnce() usize { - var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]); + var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem); const allocator = fixed_buf_alloc.allocator(); _ = std.zig.Ast.parse(allocator, source, .zig) catch @panic("parse failure"); return fixed_buf_alloc.end_index; diff --git a/lib/std/zig/system/linux.zig b/lib/std/zig/system/linux.zig index 8044e1969d4c45cd1839e24b072a7691c9d5b938..df70e71f5b3a2dc639b74031c6e51b5902f22605 100644 --- a/lib/std/zig/system/linux.zig +++ b/lib/std/zig/system/linux.zig @@ -1,7 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); const mem = std.mem; -const io = std.io; const fs = std.fs; const fmt = std.fmt; const testing = std.testing; @@ -344,8 +343,8 @@ fn testParser( expected_model: *const Target.Cpu.Model, input: []const u8, ) !void { - var fbs = io.fixedBufferStream(input); - const result = try parser.parse(arch, fbs.reader()); + var r: std.Io.Reader = .fixed(input); + const result = try parser.parse(arch, &r); try testing.expectEqual(expected_model, result.?.model); try testing.expect(expected_model.features.eql(result.?.features)); } @@ -357,20 +356,17 @@ fn testParser( // When all the lines have been analyzed the finalize method is called. fn CpuinfoParser(comptime impl: anytype) type { return struct { - fn parse(arch: Target.Cpu.Arch, reader: anytype) anyerror!?Target.Cpu { - var line_buf: [1024]u8 = undefined; + fn parse(arch: Target.Cpu.Arch, reader: *std.Io.Reader) !?Target.Cpu { var obj: impl = .{}; - - while (true) { - const line = (try reader.readUntilDelimiterOrEof(&line_buf, '\n')) orelse break; + while (reader.takeDelimiterExclusive('\n')) |line| { const colon_pos = mem.indexOfScalar(u8, line, ':') orelse continue; const key = mem.trimEnd(u8, line[0..colon_pos], " \t"); const value = mem.trimStart(u8, line[colon_pos + 1 ..], " \t"); - - if (!try obj.line_hook(key, value)) - break; + if (!try obj.line_hook(key, value)) break; + } else |err| switch (err) { + error.EndOfStream => {}, + else => |e| return e, } - return obj.finalize(arch); } }; @@ -383,15 +379,18 @@ inline fn getAArch64CpuFeature(comptime feat_reg: []const u8) u64 { } pub fn detectNativeCpuAndFeatures() ?Target.Cpu { - var f = fs.openFileAbsolute("/proc/cpuinfo", .{}) catch |err| switch (err) { + var file = fs.openFileAbsolute("/proc/cpuinfo", .{}) catch |err| switch (err) { else => return null, }; - defer f.close(); + defer file.close(); + + var buffer: [4096]u8 = undefined; // "flags" lines can get pretty long. + var file_reader = file.reader(&buffer); const current_arch = builtin.cpu.arch; switch (current_arch) { .arm, .armeb, .thumb, .thumbeb => { - return ArmCpuinfoParser.parse(current_arch, f.deprecatedReader()) catch null; + return ArmCpuinfoParser.parse(current_arch, &file_reader.interface) catch null; }, .aarch64, .aarch64_be => { const registers = [12]u64{ @@ -413,13 +412,13 @@ pub fn detectNativeCpuAndFeatures() ?Target.Cpu { return core; }, .sparc64 => { - return SparcCpuinfoParser.parse(current_arch, f.deprecatedReader()) catch null; + return SparcCpuinfoParser.parse(current_arch, &file_reader.interface) catch null; }, .powerpc, .powerpcle, .powerpc64, .powerpc64le => { - return PowerpcCpuinfoParser.parse(current_arch, f.deprecatedReader()) catch null; + return PowerpcCpuinfoParser.parse(current_arch, &file_reader.interface) catch null; }, .riscv64, .riscv32 => { - return RiscvCpuinfoParser.parse(current_arch, f.deprecatedReader()) catch null; + return RiscvCpuinfoParser.parse(current_arch, &file_reader.interface) catch null; }, else => {}, } diff --git a/src/Compilation.zig b/src/Compilation.zig index 2e20c6edbee6ad9a6740df86ecf7007dd19638f1..469a13b4a9ec907373d70175f1efecbaca9150d5 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -12,6 +12,7 @@ const ThreadPool = std.Thread.Pool; const WaitGroup = std.Thread.WaitGroup; const ErrorBundle = std.zig.ErrorBundle; const fatal = std.process.fatal; +const Writer = std.io.Writer; const Value = @import("Value.zig"); const Type = @import("Type.zig"); @@ -44,6 +45,8 @@ const Builtin = @import("Builtin.zig"); const LlvmObject = @import("codegen/llvm.zig").Object; const dev = @import("dev.zig"); +const DeprecatedLinearFifo = @import("deprecated.zig").LinearFifo; + pub const Config = @import("Compilation/Config.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. @@ -121,15 +124,15 @@ work_queues: [ } break :len len; } -]std.fifo.LinearFifo(Job, .Dynamic), +]DeprecatedLinearFifo(Job), /// These jobs are to invoke the Clang compiler to create an object file, which /// gets linked with the Compilation. -c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic), +c_object_work_queue: DeprecatedLinearFifo(*CObject), /// These jobs are to invoke the RC compiler to create a compiled resource file (.res), which /// gets linked with the Compilation. -win32_resource_work_queue: if (dev.env.supports(.win32_resource)) std.fifo.LinearFifo(*Win32Resource, .Dynamic) else struct { +win32_resource_work_queue: if (dev.env.supports(.win32_resource)) DeprecatedLinearFifo(*Win32Resource) else struct { pub fn ensureUnusedCapacity(_: @This(), _: u0) error{}!void {} pub fn readItem(_: @This()) ?noreturn { return null; @@ -995,13 +998,13 @@ pub const CObject = struct { const file = fs.cwd().openFile(file_name, .{}) catch break :source_line 0; defer file.close(); - file.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; - - var line = std.ArrayList(u8).init(eb.gpa); - defer line.deinit(); - file.deprecatedReader().readUntilDelimiterArrayList(&line, '\n', 1 << 10) catch break :source_line 0; - - break :source_line try eb.addString(line.items); + var buffer: [1024]u8 = undefined; + var file_reader = file.reader(&buffer); + file_reader.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; + var aw: Writer.Allocating = .init(eb.gpa); + defer aw.deinit(); + _ = file_reader.interface.streamDelimiterEnding(&aw.writer, '\n') catch break :source_line 0; + break :source_line try eb.addString(aw.getWritten()); }; return .{ @@ -1067,11 +1070,11 @@ pub const CObject = struct { } }; + var buffer: [1024]u8 = undefined; const file = try fs.cwd().openFile(path, .{}); defer file.close(); - var br = std.io.bufferedReader(file.deprecatedReader()); - const reader = br.reader(); - var bc = std.zig.llvm.BitcodeReader.init(gpa, .{ .reader = reader.any() }); + var file_reader = file.reader(&buffer); + var bc = std.zig.llvm.BitcodeReader.init(gpa, .{ .reader = &file_reader.interface }); defer bc.deinit(); var file_names: std.AutoArrayHashMapUnmanaged(u32, []const u8) = .empty; @@ -1873,15 +1876,12 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil if (options.verbose_llvm_cpu_features) { if (options.root_mod.resolved_target.llvm_cpu_features) |cf| print: { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); - const stderr = fs.File.stderr().deprecatedWriter(); - nosuspend { - stderr.print("compilation: {s}\n", .{options.root_name}) catch break :print; - stderr.print(" target: {s}\n", .{try target.zigTriple(arena)}) catch break :print; - stderr.print(" cpu: {s}\n", .{target.cpu.model.name}) catch break :print; - stderr.print(" features: {s}\n", .{cf}) catch {}; - } + const stderr_w = std.debug.lockStderrWriter(&.{}); + defer std.debug.unlockStderrWriter(); + stderr_w.print("compilation: {s}\n", .{options.root_name}) catch break :print; + stderr_w.print(" target: {s}\n", .{try target.zigTriple(arena)}) catch break :print; + stderr_w.print(" cpu: {s}\n", .{target.cpu.model.name}) catch break :print; + stderr_w.print(" features: {s}\n", .{cf}) catch {}; } } @@ -2483,7 +2483,7 @@ pub fn destroy(comp: *Compilation) void { if (comp.zcu) |zcu| zcu.deinit(); comp.cache_use.deinit(); - for (comp.work_queues) |work_queue| work_queue.deinit(); + for (&comp.work_queues) |*work_queue| work_queue.deinit(); comp.c_object_work_queue.deinit(); comp.win32_resource_work_queue.deinit(); @@ -3931,11 +3931,12 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle { // This AU is referenced and has a transitive compile error, meaning it referenced something with a compile error. // However, we haven't reported any such error. // This is a compiler bug. - const stderr = fs.File.stderr().deprecatedWriter(); - try stderr.writeAll("referenced transitive analysis errors, but none actually emitted\n"); - try stderr.print("{f} [transitive failure]\n", .{zcu.fmtAnalUnit(failed_unit)}); + var stderr_w = std.debug.lockStderrWriter(&.{}); + defer std.debug.unlockStderrWriter(); + try stderr_w.writeAll("referenced transitive analysis errors, but none actually emitted\n"); + try stderr_w.print("{f} [transitive failure]\n", .{zcu.fmtAnalUnit(failed_unit)}); while (ref) |r| { - try stderr.print("referenced by: {f}{s}\n", .{ + try stderr_w.print("referenced by: {f}{s}\n", .{ zcu.fmtAnalUnit(r.referencer), if (zcu.transitive_failed_analysis.contains(r.referencer)) " [transitive failure]" else "", }); @@ -5849,7 +5850,9 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr try child.spawn(); - const stderr = try child.stderr.?.deprecatedReader().readAllAlloc(arena, std.math.maxInt(usize)); + var small_buffer: [1]u8 = undefined; + var stderr_reader = child.stderr.?.reader(&small_buffer); + const stderr = try stderr_reader.interface.allocRemaining(arena, .unlimited); const term = child.wait() catch |err| { return comp.failCObj(c_object, "failed to spawn zig clang {s}: {s}", .{ argv.items[0], @errorName(err) }); @@ -6213,13 +6216,10 @@ fn spawnZigRc( const stdout = poller.fifo(.stdout); poll: while (true) { - while (stdout.readableLength() < @sizeOf(std.zig.Server.Message.Header)) { - if (!(try poller.poll())) break :poll; - } - const header = stdout.reader().readStruct(std.zig.Server.Message.Header) catch unreachable; - while (stdout.readableLength() < header.bytes_len) { - if (!(try poller.poll())) break :poll; - } + while (stdout.readableLength() < @sizeOf(std.zig.Server.Message.Header)) if (!try poller.poll()) break :poll; + var header: std.zig.Server.Message.Header = undefined; + assert(stdout.read(std.mem.asBytes(&header)) == @sizeOf(std.zig.Server.Message.Header)); + while (stdout.readableLength() < header.bytes_len) if (!try poller.poll()) break :poll; const body = stdout.readableSliceOfLen(header.bytes_len); switch (header.tag) { @@ -7209,13 +7209,16 @@ pub fn lockAndSetMiscFailure( } pub fn dump_argv(argv: []const []const u8) void { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); - const stderr = fs.File.stderr().deprecatedWriter(); - for (argv[0 .. argv.len - 1]) |arg| { - nosuspend stderr.print("{s} ", .{arg}) catch return; + var buffer: [64]u8 = undefined; + const stderr = std.debug.lockStderrWriter(&buffer); + defer std.debug.unlockStderrWriter(); + nosuspend { + for (argv) |arg| { + stderr.writeAll(arg) catch return; + (stderr.writableArray(1) catch return)[0] = ' '; + } + stderr.buffer[stderr.end - 1] = '\n'; } - nosuspend stderr.print("{s}\n", .{argv[argv.len - 1]}) catch {}; } pub fn getZigBackend(comp: Compilation) std.builtin.CompilerBackend { diff --git a/src/deprecated.zig b/src/deprecated.zig index 1f7d5c8c25efff0b238a609453b541d5c479c63f..68c712b3b1830ed4a25c26441a3b9fe1b1585fa4 100644 --- a/src/deprecated.zig +++ b/src/deprecated.zig @@ -52,15 +52,6 @@ pub fn LinearFifo(comptime T: type) type { } } - /// Reduce allocated capacity to `size`. - pub fn shrink(self: *Self, size: usize) void { - assert(size >= self.count); - self.realign(); - self.buf = self.allocator.realloc(self.buf, size) catch |e| switch (e) { - error.OutOfMemory => return, // no problem, capacity is still correct then. - }; - } - /// Ensure that the buffer can fit at least `size` items pub fn ensureTotalCapacity(self: *Self, size: usize) !void { if (self.buf.len >= size) return; @@ -76,11 +67,6 @@ pub fn LinearFifo(comptime T: type) type { return try self.ensureTotalCapacity(math.add(usize, self.count, size) catch return error.OutOfMemory); } - /// Returns number of items currently in fifo - pub fn readableLength(self: Self) usize { - return self.count; - } - /// Returns a writable slice from the 'read' end of the fifo fn readableSliceMut(self: Self, offset: usize) []T { if (offset > self.count) return &[_]T{}; @@ -95,22 +81,6 @@ pub fn LinearFifo(comptime T: type) type { } } - /// Returns a readable slice from `offset` - pub fn readableSlice(self: Self, offset: usize) []const T { - return self.readableSliceMut(offset); - } - - pub fn readableSliceOfLen(self: *Self, len: usize) []const T { - assert(len <= self.count); - const buf = self.readableSlice(0); - if (buf.len >= len) { - return buf[0..len]; - } else { - self.realign(); - return self.readableSlice(0)[0..len]; - } - } - /// Discard first `count` items in the fifo pub fn discard(self: *Self, count: usize) void { assert(count <= self.count); @@ -143,28 +113,6 @@ pub fn LinearFifo(comptime T: type) type { return c; } - /// Read data from the fifo into `dst`, returns number of items copied. - pub fn read(self: *Self, dst: []T) usize { - var dst_left = dst; - - while (dst_left.len > 0) { - const slice = self.readableSlice(0); - if (slice.len == 0) break; - const n = @min(slice.len, dst_left.len); - @memcpy(dst_left[0..n], slice[0..n]); - self.discard(n); - dst_left = dst_left[n..]; - } - - return dst.len - dst_left.len; - } - - /// Same as `read` except it returns an error union - /// The purpose of this function existing is to match `std.io.Reader` API. - fn readFn(self: *Self, dest: []u8) error{}!usize { - return self.read(dest); - } - /// Returns number of items available in fifo pub fn writableLength(self: Self) usize { return self.buf.len - self.count; @@ -183,20 +131,6 @@ pub fn LinearFifo(comptime T: type) type { } } - /// Returns a writable buffer of at least `size` items, allocating memory as needed. - /// Use `fifo.update` once you've written data to it. - pub fn writableWithSize(self: *Self, size: usize) ![]T { - try self.ensureUnusedCapacity(size); - - // try to avoid realigning buffer - var slice = self.writableSlice(0); - if (slice.len < size) { - self.realign(); - slice = self.writableSlice(0); - } - return slice; - } - /// Update the tail location of the buffer (usually follows use of writable/writableWithSize) pub fn update(self: *Self, count: usize) void { assert(self.count + count <= self.buf.len); @@ -231,201 +165,5 @@ pub fn LinearFifo(comptime T: type) type { self.buf[tail] = item; self.update(1); } - - /// Appends the data in `src` to the fifo. - /// Allocates more memory as necessary - pub fn write(self: *Self, src: []const T) !void { - try self.ensureUnusedCapacity(src.len); - - return self.writeAssumeCapacity(src); - } - - /// Same as `write` except it returns the number of bytes written, which is always the same - /// as `bytes.len`. The purpose of this function existing is to match `std.io.Writer` API. - fn appendWrite(self: *Self, bytes: []const u8) error{OutOfMemory}!usize { - try self.write(bytes); - return bytes.len; - } - - /// Make `count` items available before the current read location - fn rewind(self: *Self, count: usize) void { - assert(self.writableLength() >= count); - - var head = self.head + (self.buf.len - count); - head &= self.buf.len - 1; - self.head = head; - self.count += count; - } - - /// Place data back into the read stream - pub fn unget(self: *Self, src: []const T) !void { - try self.ensureUnusedCapacity(src.len); - - self.rewind(src.len); - - const slice = self.readableSliceMut(0); - if (src.len < slice.len) { - @memcpy(slice[0..src.len], src); - } else { - @memcpy(slice, src[0..slice.len]); - const slice2 = self.readableSliceMut(slice.len); - @memcpy(slice2[0 .. src.len - slice.len], src[slice.len..]); - } - } - - /// Returns the item at `offset`. - /// Asserts offset is within bounds. - pub fn peekItem(self: Self, offset: usize) T { - assert(offset < self.count); - - var index = self.head + offset; - index &= self.buf.len - 1; - return self.buf[index]; - } - - pub fn toOwnedSlice(self: *Self) Allocator.Error![]T { - if (self.head != 0) self.realign(); - assert(self.head == 0); - assert(self.count <= self.buf.len); - const allocator = self.allocator; - if (allocator.resize(self.buf, self.count)) { - const result = self.buf[0..self.count]; - self.* = Self.init(allocator); - return result; - } - const new_memory = try allocator.dupe(T, self.buf[0..self.count]); - allocator.free(self.buf); - self.* = Self.init(allocator); - return new_memory; - } }; } - -test "LinearFifo(u8, .Dynamic) discard(0) from empty buffer should not error on overflow" { - var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator); - defer fifo.deinit(); - - // If overflow is not explicitly allowed this will crash in debug / safe mode - fifo.discard(0); -} - -test "LinearFifo(u8, .Dynamic)" { - var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator); - defer fifo.deinit(); - - try fifo.write("HELLO"); - try testing.expectEqual(@as(usize, 5), fifo.readableLength()); - try testing.expectEqualSlices(u8, "HELLO", fifo.readableSlice(0)); - - { - var i: usize = 0; - while (i < 5) : (i += 1) { - try fifo.write(&[_]u8{fifo.peekItem(i)}); - } - try testing.expectEqual(@as(usize, 10), fifo.readableLength()); - try testing.expectEqualSlices(u8, "HELLOHELLO", fifo.readableSlice(0)); - } - - { - try testing.expectEqual(@as(u8, 'H'), fifo.readItem().?); - try testing.expectEqual(@as(u8, 'E'), fifo.readItem().?); - try testing.expectEqual(@as(u8, 'L'), fifo.readItem().?); - try testing.expectEqual(@as(u8, 'L'), fifo.readItem().?); - try testing.expectEqual(@as(u8, 'O'), fifo.readItem().?); - } - try testing.expectEqual(@as(usize, 5), fifo.readableLength()); - - { // Writes that wrap around - try testing.expectEqual(@as(usize, 11), fifo.writableLength()); - try testing.expectEqual(@as(usize, 6), fifo.writableSlice(0).len); - fifo.writeAssumeCapacity("6 {}, .stdio => { + var stdin_reader = fs.File.stdin().reader(&stdin_buffer); + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); try serve( comp, - .stdin(), - .stdout(), + &stdin_reader.interface, + &stdout_writer.interface, test_exec_args.items, self_exe_path, arg_mode, @@ -3584,10 +3588,13 @@ fn buildOutputType( const conn = try server.accept(); defer conn.stream.close(); + var input = conn.stream.reader(&stdin_buffer); + var output = conn.stream.writer(&stdout_buffer); + try serve( comp, - .{ .handle = conn.stream.handle }, - .{ .handle = conn.stream.handle }, + input.interface(), + &output.interface, test_exec_args.items, self_exe_path, arg_mode, @@ -4053,8 +4060,8 @@ fn saveState(comp: *Compilation, incremental: bool) void { fn serve( comp: *Compilation, - in: fs.File, - out: fs.File, + in: *std.Io.Reader, + out: *std.Io.Writer, test_exec_args: []const ?[]const u8, self_exe_path: ?[]const u8, arg_mode: ArgMode, @@ -4064,12 +4071,10 @@ fn serve( const gpa = comp.gpa; var server = try Server.init(.{ - .gpa = gpa, .in = in, .out = out, .zig_version = build_options.version, }); - defer server.deinit(); var child_pid: ?std.process.Child.Id = null; @@ -5491,10 +5496,10 @@ fn jitCmd( defer comp.destroy(); if (options.server) { + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); var server: std.zig.Server = .{ - .out = fs.File.stdout(), + .out = &stdout_writer.interface, .in = undefined, // won't be receiving messages - .receive_fifo = undefined, // won't be receiving messages }; try comp.update(root_prog_node); @@ -6058,7 +6063,7 @@ fn cmdAstCheck( }; } else fs.File.stdin(); defer if (zig_source_path != null) f.close(); - var file_reader: fs.File.Reader = f.reader(&stdio_buffer); + var file_reader: fs.File.Reader = f.reader(&stdin_buffer); break :s std.zig.readSourceFileToEndAlloc(arena, &file_reader) catch |err| { fatal("unable to load file '{s}' for ast-check: {s}", .{ display_path, @errorName(err) }); }; @@ -6076,7 +6081,7 @@ fn cmdAstCheck( const tree = try Ast.parse(arena, source, mode); - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; switch (mode) { .zig => { @@ -6291,7 +6296,7 @@ fn detectNativeCpuWithLLVM( } fn printCpu(cpu: std.Target.Cpu) !void { - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; if (cpu.model.llvm_name) |llvm_name| { @@ -6340,7 +6345,7 @@ fn cmdDumpLlvmInts( const dl = tm.createTargetDataLayout(); const context = llvm.Context.create(); - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; for ([_]u16{ 1, 8, 16, 32, 64, 128, 256 }) |bits| { const int_type = context.intType(bits); @@ -6369,9 +6374,8 @@ fn cmdDumpZir( defer f.close(); const zir = try Zcu.loadZirCache(arena, f); - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; - { const instruction_bytes = zir.instructions.len * // Here we don't use @sizeOf(Zir.Inst.Data) because it would include @@ -6417,7 +6421,7 @@ fn cmdChangelist( var f = fs.cwd().openFile(old_source_path, .{}) catch |err| fatal("unable to open old source file '{s}': {s}", .{ old_source_path, @errorName(err) }); defer f.close(); - var file_reader: fs.File.Reader = f.reader(&stdio_buffer); + var file_reader: fs.File.Reader = f.reader(&stdin_buffer); break :source std.zig.readSourceFileToEndAlloc(arena, &file_reader) catch |err| fatal("unable to read old source file '{s}': {s}", .{ old_source_path, @errorName(err) }); }; @@ -6425,7 +6429,7 @@ fn cmdChangelist( var f = fs.cwd().openFile(new_source_path, .{}) catch |err| fatal("unable to open new source file '{s}': {s}", .{ new_source_path, @errorName(err) }); defer f.close(); - var file_reader: fs.File.Reader = f.reader(&stdio_buffer); + var file_reader: fs.File.Reader = f.reader(&stdin_buffer); break :source std.zig.readSourceFileToEndAlloc(arena, &file_reader) catch |err| fatal("unable to read new source file '{s}': {s}", .{ new_source_path, @errorName(err) }); }; @@ -6457,7 +6461,7 @@ fn cmdChangelist( var inst_map: std.AutoHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .empty; try Zcu.mapOldZirToNew(arena, old_zir, new_zir, &inst_map); - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; { try stdout_bw.print("Instruction mappings:\n", .{}); @@ -6917,7 +6921,7 @@ fn cmdFetch( const name = switch (save) { .no => { - var stdout = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout = fs.File.stdout().writerStreaming(&stdout_buffer); try stdout.interface.print("{s}\n", .{package_hash_slice}); try stdout.interface.flush(); return cleanExit(); -- 2.54.0 From bad836a69bf3c1da29f39ed1a6c8bf6c66febfb6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 06:45:25 -0700 Subject: [PATCH 12/26] Compilation: revert some stuff --- src/Compilation.zig | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 469a13b4a9ec907373d70175f1efecbaca9150d5..916a025bc973c20d4efba2476e0f224a2bef9acc 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -998,13 +998,13 @@ pub const CObject = struct { const file = fs.cwd().openFile(file_name, .{}) catch break :source_line 0; defer file.close(); - var buffer: [1024]u8 = undefined; - var file_reader = file.reader(&buffer); - file_reader.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; - var aw: Writer.Allocating = .init(eb.gpa); - defer aw.deinit(); - _ = file_reader.interface.streamDelimiterEnding(&aw.writer, '\n') catch break :source_line 0; - break :source_line try eb.addString(aw.getWritten()); + file.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; + + var line = std.ArrayList(u8).init(eb.gpa); + defer line.deinit(); + file.deprecatedReader().readUntilDelimiterArrayList(&line, '\n', 1 << 10) catch break :source_line 0; + + break :source_line try eb.addString(line.items); }; return .{ @@ -5850,9 +5850,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr try child.spawn(); - var small_buffer: [1]u8 = undefined; - var stderr_reader = child.stderr.?.reader(&small_buffer); - const stderr = try stderr_reader.interface.allocRemaining(arena, .unlimited); + const stderr = try child.stderr.?.deprecatedReader().readAllAlloc(arena, std.math.maxInt(usize)); const term = child.wait() catch |err| { return comp.failCObj(c_object, "failed to spawn zig clang {s}: {s}", .{ argv.items[0], @errorName(err) }); -- 2.54.0 From 83d1f88ac5f053f11b91d9c3d51f5c63355b8ca3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 10:42:04 -0700 Subject: [PATCH 13/26] std.debug: add assertAligned --- lib/std/debug.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/std/debug.zig b/lib/std/debug.zig index a7b88e4257c193a16ee3b24ccba51c4664937e47..655852c65e83ed2f446e675f53293df5dc4b81bc 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -566,6 +566,13 @@ pub fn assertReadable(slice: []const volatile u8) void { for (slice) |*byte| _ = byte.*; } +/// Invokes detectable illegal behavior when the provided array is not aligned +/// to the provided amount. +pub fn assertAligned(ptr: anytype, comptime alignment: std.mem.Alignment) void { + const aligned_ptr: *align(alignment.toByteUnits()) anyopaque = @alignCast(@ptrCast(ptr)); + _ = aligned_ptr; +} + /// Equivalent to `@panic` but with a formatted message. pub fn panic(comptime format: []const u8, args: anytype) noreturn { @branchHint(.cold); -- 2.54.0 From bd64bf0e47e75481569dcc3ba519e3d2e7f7b276 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 10:42:24 -0700 Subject: [PATCH 14/26] std.mem: add byteSwapAllElements --- lib/std/mem.zig | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 33e68eedad64d0ee6923a158cf4cf2893f44c495..1a61076f32caec5d98e008b691ae54b40d6a2cc0 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -2179,22 +2179,8 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void { const BackingInt = std.meta.Int(.unsigned, @bitSizeOf(S)); ptr.* = @bitCast(@byteSwap(@as(BackingInt, @bitCast(ptr.*)))); }, - .array => { - for (ptr) |*item| { - switch (@typeInfo(@TypeOf(item.*))) { - .@"struct", .@"union", .array => byteSwapAllFields(@TypeOf(item.*), item), - .@"enum" => { - item.* = @enumFromInt(@byteSwap(@intFromEnum(item.*))); - }, - .bool => {}, - .float => |float_info| { - item.* = @bitCast(@byteSwap(@as(std.meta.Int(.unsigned, float_info.bits), @bitCast(item.*)))); - }, - else => { - item.* = @byteSwap(item.*); - }, - } - } + .array => |info| { + byteSwapAllElements(info.child, ptr); }, else => { ptr.* = @byteSwap(ptr.*); @@ -2258,6 +2244,24 @@ test byteSwapAllFields { }, k); } +pub fn byteSwapAllElements(comptime Elem: type, slice: []Elem) void { + for (slice) |*elem| { + switch (@typeInfo(@TypeOf(elem.*))) { + .@"struct", .@"union", .array => byteSwapAllFields(@TypeOf(elem.*), elem), + .@"enum" => { + elem.* = @enumFromInt(@byteSwap(@intFromEnum(elem.*))); + }, + .bool => {}, + .float => |float_info| { + elem.* = @bitCast(@byteSwap(@as(std.meta.Int(.unsigned, float_info.bits), @bitCast(elem.*)))); + }, + else => { + elem.* = @byteSwap(elem.*); + }, + } + } +} + /// Returns an iterator that iterates over the slices of `buffer` that are not /// any of the items in `delimiters`. /// -- 2.54.0 From 8489bab1f46ef7a5cdcf598b10f1150a9e0c2f0d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 10:43:02 -0700 Subject: [PATCH 15/26] std.Io.Writer: add missing writeSliceSwap --- lib/std/Io/Writer.zig | 28 +++++++++++++++++++++++++++- lib/std/zig/Server.zig | 2 ++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 4b0e142fb0f182bca54793ebec6fdf7f7bb0a609..11bc05a00d931afa64f58a03a1bbb8342c6ec91e 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -851,6 +851,9 @@ pub inline fn writeStruct(w: *Writer, value: anytype, endian: std.builtin.Endian } } +/// If, `endian` is not native, +/// * Asserts that the buffer storage capacity is at least enough to store `@sizeOf(Elem)` +/// * Asserts that the buffer is aligned enough for `@alignOf(Elem)`. pub inline fn writeSliceEndian( w: *Writer, Elem: type, @@ -860,7 +863,22 @@ pub inline fn writeSliceEndian( if (native_endian == endian) { return writeAll(w, @ptrCast(slice)); } else { - return w.writeArraySwap(w, Elem, slice); + return writeSliceSwap(w, Elem, slice); + } +} + +/// Asserts that the buffer storage capacity is at least enough to store `@sizeOf(Elem)` +/// +/// Asserts that the buffer is aligned enough for `@alignOf(Elem)`. +pub fn writeSliceSwap(w: *Writer, Elem: type, slice: []const Elem) Error!void { + var i: usize = 0; + while (i < slice.len) { + const dest_bytes = try w.writableSliceGreedy(@sizeOf(Elem)); + const dest: []Elem = @alignCast(@ptrCast(dest_bytes[0 .. dest_bytes.len - dest_bytes.len % @sizeOf(Elem)])); + const copy_len = @min(dest.len, slice.len - i); + @memcpy(dest[0..copy_len], slice[i..][0..copy_len]); + i += copy_len; + std.mem.byteSwapAllElements(Elem, dest); } } @@ -2630,3 +2648,11 @@ test writeStruct { }, &buffer); } } + +test writeSliceEndian { + var buffer: [4]u8 align(2) = undefined; + var w: Writer = .fixed(&buffer); + const array: [2]u16 = .{ 0x1234, 0x5678 }; + try writeSliceEndian(&w, u16, &array, .big); + try testing.expectEqualSlices(u8, &.{ 0x12, 0x34, 0x56, 0x78 }, &buffer); +} diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 38ad45e1e161acb6af9fc4669653695d78c37031..8fc016d284dacc923401992d675c114bb0b0eade 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -118,6 +118,8 @@ pub fn init(options: Options) !Server { .in = options.in, .out = options.out, }; + assert(s.out.buffer.len >= 4); + std.debug.assertAligned(s.out.buffer.ptr, .@"4"); try s.serveStringMessage(.zig_version, options.zig_version); return s; } -- 2.54.0 From b956ae20af86384ef5d15b2417b0bf095b5b3740 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 11:15:12 -0700 Subject: [PATCH 16/26] frontend: align those stdio buffers --- src/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 9958c461b13e7730815b017bf22d4ae76865ba96..dc931f119bf22d0a863d5de02e5b05e87f8454cd 100644 --- a/src/main.zig +++ b/src/main.zig @@ -66,9 +66,9 @@ pub fn wasi_cwd() std.os.wasi.fd_t { const fatal = std.process.fatal; /// This can be global since stdin is a singleton. -var stdin_buffer: [4096]u8 = undefined; +var stdin_buffer: [4096]u8 align(std.heap.page_size_min) = undefined; /// This can be global since stdout is a singleton. -var stdout_buffer: [4096]u8 = undefined; +var stdout_buffer: [4096]u8 align(std.heap.page_size_min) = undefined; /// Shaming all the locations that inappropriately use an O(N) search algorithm. /// Please delete this and fix the compilation errors! -- 2.54.0 From d396780925914025c42576d45a2f668a64f7336b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 11:16:31 -0700 Subject: [PATCH 17/26] Compilation: unrevert some stuff --- src/Compilation.zig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 916a025bc973c20d4efba2476e0f224a2bef9acc..b5597017c4b2a110b9638c7a6be6035885f44808 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -998,13 +998,13 @@ pub const CObject = struct { const file = fs.cwd().openFile(file_name, .{}) catch break :source_line 0; defer file.close(); - file.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; - - var line = std.ArrayList(u8).init(eb.gpa); - defer line.deinit(); - file.deprecatedReader().readUntilDelimiterArrayList(&line, '\n', 1 << 10) catch break :source_line 0; - - break :source_line try eb.addString(line.items); + var buffer: [1024]u8 = undefined; + var file_reader = file.reader(&buffer); + file_reader.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; + var aw: Writer.Allocating = .init(eb.gpa); + defer aw.deinit(); + _ = file_reader.interface.streamDelimiterEnding(&aw.writer, '\n') catch break :source_line 0; + break :source_line try eb.addString(aw.getWritten()); }; return .{ -- 2.54.0 From 83960e0eb068f786c46c3fe559016e1e9faea3cd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 17:38:13 -0700 Subject: [PATCH 18/26] disable -fno-llvm -target wasm32-wasi testing no active maintainer, and it's failing to lower some basic stuff --- test/src/Cases.zig | 2 -- test/tests.zig | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/test/src/Cases.zig b/test/src/Cases.zig index 522fe6b38557eb9cf75c8149055abffc875bf791..a92f9fd158d3bc5dbf3831150c957aab0b961a28 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -800,8 +800,6 @@ const TestManifestConfigDefaults = struct { } // Windows defaults = defaults ++ "x86_64-windows" ++ ","; - // Wasm - defaults = defaults ++ "wasm32-wasi"; break :blk defaults; }; } else if (std.mem.eql(u8, key, "output_mode")) { diff --git a/test/tests.zig b/test/tests.zig index 2a6c4166fa518de077b962e9cc3e1d2ed3e2dd85..db4407172f96384b7d87e3cfc2f080bcaceebb4e 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -1335,15 +1335,16 @@ const test_targets = blk: { // WASI Targets - .{ - .target = .{ - .cpu_arch = .wasm32, - .os_tag = .wasi, - .abi = .none, - }, - .use_llvm = false, - .use_lld = false, - }, + // TODO: lowerTry for pointers + //.{ + // .target = .{ + // .cpu_arch = .wasm32, + // .os_tag = .wasi, + // .abi = .none, + // }, + // .use_llvm = false, + // .use_lld = false, + //}, .{ .target = .{ .cpu_arch = .wasm32, -- 2.54.0 From 741a66e03cd8644d01b38849f2bd8f70cae6beca Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 20:51:37 -0700 Subject: [PATCH 19/26] std.zig.llvm.BitcodeReader: fix 32-bit skipBlock --- lib/std/zig/llvm/BitcodeReader.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/zig/llvm/BitcodeReader.zig b/lib/std/zig/llvm/BitcodeReader.zig index f1d47e93c0139c9a16f5a994e96918f8a564d0c5..f691f12a8c9cbab61f1a1c454dc4acdafbec54cd 100644 --- a/lib/std/zig/llvm/BitcodeReader.zig +++ b/lib/std/zig/llvm/BitcodeReader.zig @@ -177,7 +177,7 @@ pub fn next(bc: *BitcodeReader) !?Item { pub fn skipBlock(bc: *BitcodeReader, block: Block) !void { assert(bc.bit_offset == 0); - try bc.reader.discardAll(4 * @as(u34, block.len)); + try bc.reader.discardAll(4 * @as(usize, block.len)); try bc.endBlock(); } -- 2.54.0 From b4fd57a9c114748afb9ba0a04bede61089a02ddf Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sat, 19 Jul 2025 18:26:56 -0400 Subject: [PATCH 20/26] llvm: workaround crashes in llvm loop optimizations Workaround for #24383 --- src/codegen/llvm.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index b3391cfb6ac7e09c975f21af21cd8dd1871545f8..a570dd5ec0cd37127a1066b9ce09446ef058789d 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -6385,6 +6385,9 @@ pub const FuncGen = struct { // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null; + // Workaround for https://github.com/ziglang/zig/issues/24383: + if (self.ng.ownerModule().optimize_mode == .ReleaseSafe) break :jmp_table null; + // On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just // about acceptable - it won't fill L1d cache on most CPUs. const max_table_len = 1024; -- 2.54.0 From c40fb96ca358e2ef28aecc2b7ebc5ffab43ccac8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 21:53:48 -0700 Subject: [PATCH 21/26] std.Io.Writer: fix writeSliceSwap tried to be too clever, wrote bad code --- lib/compiler/test_runner.zig | 12 ++++++------ lib/std/Build/Step/Run.zig | 2 +- lib/std/Io/Reader.zig | 34 +++++++++++++++++----------------- lib/std/Io/Writer.zig | 23 +++++++---------------- lib/std/zig/Server.zig | 6 ++---- src/Zcu.zig | 2 +- src/Zcu/PerThread.zig | 2 +- 7 files changed, 35 insertions(+), 46 deletions(-) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index a69066f09cb9f2de19bd3c8cb4ad6fef63ef0fe2..8b60a75399a8c06c95143a4417a5a55774afb5b1 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -10,10 +10,10 @@ pub const std_options: std.Options = .{ }; var log_err_count: usize = 0; -var fba_buffer: [8192]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); -var stdin_buffer: [std.heap.page_size_min]u8 align(std.heap.page_size_min) = undefined; -var stdout_buffer: [std.heap.page_size_min]u8 align(std.heap.page_size_min) = undefined; +var fba_buffer: [8192]u8 = undefined; +var stdin_buffer: [4096]u8 = undefined; +var stdout_buffer: [4096]u8 = undefined; const crippled = switch (builtin.zig_backend) { .stage2_powerpc, @@ -68,8 +68,8 @@ pub fn main() void { fn mainServer() !void { @disableInstrumentation(); - var stdin_reader = std.fs.File.stdin().reader(&stdin_buffer); - var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + var stdin_reader = std.fs.File.stdin().readerStreaming(&stdin_buffer); + var stdout_writer = std.fs.File.stdout().writerStreaming(&stdout_buffer); var server = try std.zig.Server.init(.{ .in = &stdin_reader.interface, .out = &stdout_writer.interface, @@ -104,7 +104,7 @@ fn mainServer() !void { defer testing.allocator.free(expected_panic_msgs); for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| { - name.* = @as(u32, @intCast(string_bytes.items.len)); + name.* = @intCast(string_bytes.items.len); try string_bytes.ensureUnusedCapacity(testing.allocator, test_fn.name.len + 1); string_bytes.appendSliceAssumeCapacity(test_fn.name); string_bytes.appendAssumeCapacity(0); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index b4e2e73c0a0b8a5e60a61e34103edf2c5a297e5c..1742da33c8ffffc0015141995ab78ceeff9774bb 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -1742,7 +1742,7 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { .tag = tag, .bytes_len = 0, }; - try file.writeAll(std.mem.asBytes(&header)); + try file.writeAll(@ptrCast(&header)); } fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index: u32) !void { diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index f2a1ec7287bc0998a631486078eeafe5d79130c7..497b906c978841802498f69997fc7126a5641a74 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -1108,9 +1108,9 @@ pub fn takeVarInt(r: *Reader, comptime Int: type, endian: std.builtin.Endian, n: /// Asserts the buffer was initialized with a capacity at least `@sizeOf(T)`. /// /// See also: -/// * `peekStructReference` +/// * `peekStructPointer` /// * `takeStruct` -pub fn takeStructReference(r: *Reader, comptime T: type) Error!*align(1) T { +pub fn takeStructPointer(r: *Reader, comptime T: type) Error!*align(1) T { // Only extern and packed structs have defined in-memory layout. comptime assert(@typeInfo(T).@"struct".layout != .auto); return @ptrCast(try r.takeArray(@sizeOf(T))); @@ -1122,9 +1122,9 @@ pub fn takeStructReference(r: *Reader, comptime T: type) Error!*align(1) T { /// Asserts the buffer was initialized with a capacity at least `@sizeOf(T)`. /// /// See also: -/// * `takeStructReference` +/// * `takeStructPointer` /// * `peekStruct` -pub fn peekStructReference(r: *Reader, comptime T: type) Error!*align(1) T { +pub fn peekStructPointer(r: *Reader, comptime T: type) Error!*align(1) T { // Only extern and packed structs have defined in-memory layout. comptime assert(@typeInfo(T).@"struct".layout != .auto); return @ptrCast(try r.peekArray(@sizeOf(T))); @@ -1136,14 +1136,14 @@ pub fn peekStructReference(r: *Reader, comptime T: type) Error!*align(1) T { /// when `endian` is comptime-known and matches the host endianness. /// /// See also: -/// * `takeStructReference` +/// * `takeStructPointer` /// * `peekStruct` pub inline fn takeStruct(r: *Reader, comptime T: type, endian: std.builtin.Endian) Error!T { switch (@typeInfo(T)) { .@"struct" => |info| switch (info.layout) { .auto => @compileError("ill-defined memory layout"), .@"extern" => { - var res = (try r.takeStructReference(T)).*; + var res = (try r.takeStructPointer(T)).*; if (native_endian != endian) std.mem.byteSwapAllFields(T, &res); return res; }, @@ -1162,13 +1162,13 @@ pub inline fn takeStruct(r: *Reader, comptime T: type, endian: std.builtin.Endia /// /// See also: /// * `takeStruct` -/// * `peekStructReference` +/// * `peekStructPointer` pub inline fn peekStruct(r: *Reader, comptime T: type, endian: std.builtin.Endian) Error!T { switch (@typeInfo(T)) { .@"struct" => |info| switch (info.layout) { .auto => @compileError("ill-defined memory layout"), .@"extern" => { - var res = (try r.peekStructReference(T)).*; + var res = (try r.peekStructPointer(T)).*; if (native_endian != endian) std.mem.byteSwapAllFields(T, &res); return res; }, @@ -1557,27 +1557,27 @@ test takeVarInt { try testing.expectError(error.EndOfStream, r.takeVarInt(u16, .little, 1)); } -test takeStructReference { +test takeStructPointer { var r: Reader = .fixed(&.{ 0x12, 0x00, 0x34, 0x56 }); const S = extern struct { a: u8, b: u16 }; switch (native_endian) { - .little => try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.takeStructReference(S)).*), - .big => try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.takeStructReference(S)).*), + .little => try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.takeStructPointer(S)).*), + .big => try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.takeStructPointer(S)).*), } - try testing.expectError(error.EndOfStream, r.takeStructReference(S)); + try testing.expectError(error.EndOfStream, r.takeStructPointer(S)); } -test peekStructReference { +test peekStructPointer { var r: Reader = .fixed(&.{ 0x12, 0x00, 0x34, 0x56 }); const S = extern struct { a: u8, b: u16 }; switch (native_endian) { .little => { - try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.peekStructReference(S)).*); - try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.peekStructReference(S)).*); + try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.peekStructPointer(S)).*); + try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.peekStructPointer(S)).*); }, .big => { - try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.peekStructReference(S)).*); - try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.peekStructReference(S)).*); + try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.peekStructPointer(S)).*); + try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.peekStructPointer(S)).*); }, } } diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 11bc05a00d931afa64f58a03a1bbb8342c6ec91e..4d8f04b2467493b1ab322ce015ba64adb25aae70 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -851,9 +851,6 @@ pub inline fn writeStruct(w: *Writer, value: anytype, endian: std.builtin.Endian } } -/// If, `endian` is not native, -/// * Asserts that the buffer storage capacity is at least enough to store `@sizeOf(Elem)` -/// * Asserts that the buffer is aligned enough for `@alignOf(Elem)`. pub inline fn writeSliceEndian( w: *Writer, Elem: type, @@ -867,18 +864,11 @@ pub inline fn writeSliceEndian( } } -/// Asserts that the buffer storage capacity is at least enough to store `@sizeOf(Elem)` -/// -/// Asserts that the buffer is aligned enough for `@alignOf(Elem)`. pub fn writeSliceSwap(w: *Writer, Elem: type, slice: []const Elem) Error!void { - var i: usize = 0; - while (i < slice.len) { - const dest_bytes = try w.writableSliceGreedy(@sizeOf(Elem)); - const dest: []Elem = @alignCast(@ptrCast(dest_bytes[0 .. dest_bytes.len - dest_bytes.len % @sizeOf(Elem)])); - const copy_len = @min(dest.len, slice.len - i); - @memcpy(dest[0..copy_len], slice[i..][0..copy_len]); - i += copy_len; - std.mem.byteSwapAllElements(Elem, dest); + for (slice) |elem| { + var tmp = elem; + std.mem.byteSwapAllFields(Elem, &tmp); + try w.writeAll(@ptrCast(&tmp)); } } @@ -2650,9 +2640,10 @@ test writeStruct { } test writeSliceEndian { - var buffer: [4]u8 align(2) = undefined; + var buffer: [5]u8 align(2) = undefined; var w: Writer = .fixed(&buffer); + try w.writeByte('x'); const array: [2]u16 = .{ 0x1234, 0x5678 }; try writeSliceEndian(&w, u16, &array, .big); - try testing.expectEqualSlices(u8, &.{ 0x12, 0x34, 0x56, 0x78 }, &buffer); + try testing.expectEqualSlices(u8, &.{ 'x', 0x12, 0x34, 0x56, 0x78 }, &buffer); } diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 8fc016d284dacc923401992d675c114bb0b0eade..12bd259b16d8b3ab01ac365f052c8e0bd5f9b629 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -118,8 +118,6 @@ pub fn init(options: Options) !Server { .in = options.in, .out = options.out, }; - assert(s.out.buffer.len >= 4); - std.debug.assertAligned(s.out.buffer.ptr, .@"4"); try s.serveStringMessage(.zig_version, options.zig_version); return s; } @@ -203,8 +201,8 @@ pub const TestMetadata = struct { pub fn serveTestMetadata(s: *Server, test_metadata: TestMetadata) !void { const header: OutMessage.TestMetadata = .{ - .tests_len = @as(u32, @intCast(test_metadata.names.len)), - .string_bytes_len = @as(u32, @intCast(test_metadata.string_bytes.len)), + .tests_len = @intCast(test_metadata.names.len), + .string_bytes_len = @intCast(test_metadata.string_bytes.len), }; const trailing = 2; const bytes_len = @sizeOf(OutMessage.TestMetadata) + diff --git a/src/Zcu.zig b/src/Zcu.zig index 897bb6e89eb99fef3b29c69f6348fa52c8367f19..d337f0b943f7c36666b7dcc63ea81404c390693d 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -2821,7 +2821,7 @@ pub fn loadZirCache(gpa: Allocator, cache_file: std.fs.File) !Zir { var buffer: [2000]u8 = undefined; var file_reader = cache_file.reader(&buffer); return result: { - const header = file_reader.interface.takeStructReference(Zir.Header) catch |err| break :result err; + const header = file_reader.interface.takeStructPointer(Zir.Header) catch |err| break :result err; break :result loadZirCacheBody(gpa, header.*, &file_reader.interface); } catch |err| switch (err) { error.ReadFailed => return file_reader.err.?, diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index dc2308add102f0398ff395af79e8023ef5154abb..26f008e1c8880a497bfe5e0b5984c15f2eff690b 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -349,7 +349,7 @@ fn loadZirZoirCache( const cache_br = &cache_fr.interface; // First we read the header to determine the lengths of arrays. - const header = (cache_br.takeStructReference(Header) catch |err| switch (err) { + const header = (cache_br.takeStructPointer(Header) catch |err| switch (err) { error.ReadFailed => return cache_fr.err.?, // This can happen if Zig bails out of this function between creating // the cached file and writing it. -- 2.54.0 From c58cce799932a5b9a735ac359794ec8bcde61633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= Date: Sun, 20 Jul 2025 01:15:46 +0200 Subject: [PATCH 22/26] std.Build.Step.Run: fix up 681d324c49e7cdc773cc891ea49ed69dd03c23c7 https://github.com/ziglang/zig/pull/24151/files#r2217494741 --- lib/std/Build/Step/Run.zig | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index b4e2e73c0a0b8a5e60a61e34103edf2c5a297e5c..b8d1a3a9cbcf186d8b285ce8a1a152101bd4bd89 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -1118,10 +1118,12 @@ fn runCommand( // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. if (env_map.get("WINEDEBUG") == null) { - // We don't own `env_map` at this point, so turn it into a copy before modifying it. - env_map = arena.create(EnvMap) catch @panic("OOM"); - env_map.hash_map = try env_map.hash_map.cloneWithAllocator(arena); - try env_map.put("WINEDEBUG", "-all"); + // We don't own `env_map` at this point, so create a copy in order to modify it. + const new_env_map = arena.create(EnvMap) catch @panic("OOM"); + new_env_map.hash_map = try env_map.hash_map.cloneWithAllocator(arena); + try new_env_map.put("WINEDEBUG", "-all"); + + env_map = new_env_map; } } else { return failForeign(run, "-fwine", argv[0], exe); -- 2.54.0 From 4780cc50cf7e42f6af3eb71ef3897f4b341215b4 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Sun, 20 Jul 2025 13:57:31 +1200 Subject: [PATCH 23/26] std.Io.Writer: support alignment for `{t}` specifier --- lib/std/Io/Writer.zig | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 4b0e142fb0f182bca54793ebec6fdf7f7bb0a609..98fec6196f62f85a5408a4b6fbc74e32911bef36 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -1123,8 +1123,8 @@ pub fn printValue( else => invalidFmtError(fmt, value), }, 't' => switch (@typeInfo(T)) { - .error_set => return w.writeAll(@errorName(value)), - .@"enum", .@"union" => return w.writeAll(@tagName(value)), + .error_set => return w.alignBufferOptions(@errorName(value), options), + .@"enum", .@"union" => return w.alignBufferOptions(@tagName(value), options), else => invalidFmtError(fmt, value), }, else => {}, @@ -2134,6 +2134,14 @@ test "bytes.hex" { try testing.expectFmt("lowercase: 000ebabe\n", "lowercase: {x}\n", .{bytes_with_zeros}); } +test "padding" { + const foo: enum { foo } = .foo; + try testing.expectFmt("tag: |foo |\n", "tag: |{t:<4}|\n", .{foo}); + + const bar: error{bar} = error.bar; + try testing.expectFmt("error: |bar |\n", "error: |{t:<4}|\n", .{bar}); +} + test fixed { { var buf: [255]u8 = undefined; -- 2.54.0 From 14bb533203b87d7ef8c7d9434f8efba469ed1b0f Mon Sep 17 00:00:00 2001 From: antlilja Date: Sun, 20 Jul 2025 13:48:24 +0200 Subject: [PATCH 24/26] use stdout_buffer instead of stdio_buffer in main.zig --- src/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index d55ec85b90a95f9b9584869c6ecd3c08f5585fc0..7ad40e1a68a136d53d191d9e6b67195cd63457ff 100644 --- a/src/main.zig +++ b/src/main.zig @@ -346,7 +346,7 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "targets")) { dev.check(.targets_command); const host = std.zig.resolveTargetQueryOrFatal(.{}); - var stdout_writer = fs.File.stdout().writer(&stdio_buffer); + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); try @import("print_targets.zig").cmdTargets(arena, cmd_args, &stdout_writer.interface, &host); return stdout_writer.interface.flush(); } else if (mem.eql(u8, cmd, "version")) { @@ -359,7 +359,7 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "env")) { dev.check(.env_command); verifyLibcxxCorrectlyLinked(); - var stdout_writer = fs.File.stdout().writer(&stdio_buffer); + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); try @import("print_env.zig").cmdEnv(arena, &stdout_writer.interface); return stdout_writer.interface.flush(); } else if (mem.eql(u8, cmd, "reduce")) { -- 2.54.0 From f657767b600064c520fac0e72da6016508a8b601 Mon Sep 17 00:00:00 2001 From: Kendall Condon Date: Sun, 20 Jul 2025 13:30:57 -0400 Subject: [PATCH 25/26] langref: upgrade grammar.y for asm clobber change --- doc/langref.html.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index dcf13e812d7ad056387c1879cc94f94226127089..e8189e5c420ce42559b313b9b16090a5c8b94455 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -7987,7 +7987,7 @@ AsmInput <- COLON AsmInputList AsmClobbers? AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN -AsmClobbers <- COLON StringList +AsmClobbers <- COLON Expr # *** Helper grammar *** BreakLabel <- COLON IDENTIFIER -- 2.54.0 From e4abdf5a133ac4822644da1dabd5437ac751d78d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 20 Jul 2025 11:23:12 -0700 Subject: [PATCH 26/26] std.Io.Reader: fix takeStruct/peekStruct packed closes #24516 --- lib/std/Io/Reader.zig | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 659ebf2f977c48e28fb6477a476b04b33be96afa..f25e11352234960c4a9ac7246a654a3e6f2a8be9 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -1148,7 +1148,7 @@ pub inline fn takeStruct(r: *Reader, comptime T: type, endian: std.builtin.Endia return res; }, .@"packed" => { - return takeInt(r, info.backing_integer.?, endian); + return @bitCast(try takeInt(r, info.backing_integer.?, endian)); }, }, else => @compileError("not a struct"), @@ -1173,7 +1173,7 @@ pub inline fn peekStruct(r: *Reader, comptime T: type, endian: std.builtin.Endia return res; }, .@"packed" => { - return peekInt(r, info.backing_integer.?, endian); + return @bitCast(try peekInt(r, info.backing_integer.?, endian)); }, }, else => @compileError("not a struct"), @@ -1724,6 +1724,27 @@ test "takeDelimiterInclusive when it rebases" { } } +test "takeStruct and peekStruct packed" { + var r: Reader = .fixed(&.{ 0b11110000, 0b00110011 }); + const S = packed struct(u16) { a: u2, b: u6, c: u7, d: u1 }; + + try testing.expectEqual(@as(S, .{ + .a = 0b11, + .b = 0b001100, + .c = 0b1110000, + .d = 0b1, + }), try r.peekStruct(S, .big)); + + try testing.expectEqual(@as(S, .{ + .a = 0b11, + .b = 0b001100, + .c = 0b1110000, + .d = 0b1, + }), try r.takeStruct(S, .big)); + + try testing.expectError(error.EndOfStream, r.takeStruct(S, .little)); +} + /// Provides a `Reader` implementation by passing data from an underlying /// reader through `Hasher.update`. /// -- 2.54.0